mainloop.c

/* Copyright (c) 2001 Matej Pfajfar.
 * Copyright (c) 2001-2004, Roger Dingledine.
 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 * Copyright (c) 2007-2019, The Tor Project, Inc. */
/* See LICENSE for licensing information */

/**
 * \file mainloop.c
 * \brief Toplevel module. Handles signals, multiplexes between
 *     connections, implements main loop, and drives scheduled events.
 *
 * For the main loop itself; see run_main_loop_once().  It invokes the rest of
 * Tor mostly through Libevent callbacks.  Libevent callbacks can happen when
 * a timer elapses, a signal is received, a socket is ready to read or write,
 * or an event is manually activated.
 *
 * Most events in Tor are driven from these callbacks:
 *  <ul>
 *   <li>conn_read_callback() and conn_write_callback() here, which are
 *     invoked when a socket is ready to read or write respectively.
 *   <li>signal_callback(), which handles incoming signals.
 *  </ul>
 * Other events are used for specific purposes, or for building more complex
 * control structures.  If you search for usage of tor_libevent_new(), you
 * will find all the events that we construct in Tor.
 *
 * Tor has numerous housekeeping operations that need to happen
 * regularly. They are handled in different ways:
 * <ul>
 *   <li>The most frequent operations are handled after every read or write
 *    event, at the end of connection_handle_read() and
 *    connection_handle_write().
 *
 *   <li>The next most frequent operations happen after each invocation of the
 *     main loop, in run_main_loop_once().
 *
 *   <li>Once per second, we run all of the operations listed in
 *     second_elapsed_callback(), and in its child, run_scheduled_events().
 *
 *   <li>Once-a-second operations are handled in second_elapsed_callback().
 *
 *   <li>More infrequent operations take place based on the periodic event
 *     driver in periodic.c .  These are stored in the periodic_events[]
 *     table.
 * </ul>
 *
 **/

#define MAINLOOP_PRIVATE
#include "core/or/or.h"

#include "app/config/config.h"
#include "app/config/statefile.h"
#include "app/main/ntmain.h"
#include "core/mainloop/connection.h"
#include "core/mainloop/cpuworker.h"
#include "core/mainloop/mainloop.h"
#include "core/mainloop/netstatus.h"
#include "core/mainloop/periodic.h"
#include "core/or/channel.h"
#include "core/or/channelpadding.h"
#include "core/or/channeltls.h"
#include "core/or/circuitbuild.h"
#include "core/or/circuitlist.h"
#include "core/or/circuituse.h"
#include "core/or/connection_edge.h"
#include "core/or/connection_or.h"
#include "core/or/dos.h"
#include "core/or/status.h"
#include "feature/client/addressmap.h"
#include "feature/client/bridges.h"
#include "feature/client/dnsserv.h"
#include "feature/client/entrynodes.h"
#include "feature/client/transports.h"
#include "feature/control/control.h"
#include "feature/dirauth/authmode.h"
#include "feature/dirauth/reachability.h"
#include "feature/dircache/consdiffmgr.h"
#include "feature/dircache/dirserv.h"
#include "feature/dircommon/directory.h"
#include "feature/hibernate/hibernate.h"
#include "feature/hs/hs_cache.h"
#include "feature/hs/hs_client.h"
#include "feature/hs/hs_service.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/relay/dns.h"
#include "feature/relay/routerkeys.h"
#include "feature/relay/routermode.h"
#include "feature/relay/selftest.h"
#include "feature/rend/rendcache.h"
#include "feature/rend/rendservice.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/predict_ports.h"
#include "feature/stats/rephist.h"
#include "lib/buf/buffers.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/err/backtrace.h"
#include "lib/tls/buffers_tls.h"

#include "lib/net/buffers_net.h"
#include "lib/evloop/compat_libevent.h"

#include <event2/event.h>

#include "feature/dirauth/dirvote.h"
#include "feature/dirauth/authmode.h"

#include "core/or/cell_st.h"
#include "core/or/entry_connection_st.h"
#include "feature/nodelist/networkstatus_st.h"
#include "core/or/or_connection_st.h"
#include "app/config/or_state_st.h"
#include "feature/nodelist/routerinfo_st.h"
#include "core/or/socks_request_st.h"

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

#ifdef HAVE_SYSTEMD
#   if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__)
/* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse
 * Coverity. Here's a kludge to unconfuse it.
 */
#   define __INCLUDE_LEVEL__ 2
#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
#include <systemd/sd-daemon.h>
#endif /* defined(HAVE_SYSTEMD) */

/* Token bucket for all traffic. */
token_bucket_rw_t global_bucket;

/* Token bucket for relayed traffic. */
token_bucket_rw_t global_relayed_bucket;

/* XXX we might want to keep stats about global_relayed_*_bucket too. Or not.*/
/** How many bytes have we read since we started the process? */
static uint64_t stats_n_bytes_read = 0;
/** How many bytes have we written since we started the process? */
static uint64_t stats_n_bytes_written = 0;
/** What time did this process start up? */
time_t time_of_process_start = 0;
/** How many seconds have we been running? */
static long stats_n_seconds_working = 0;
/** How many times have we returned from the main loop successfully? */
static uint64_t stats_n_main_loop_successes = 0;
/** How many times have we received an error from the main loop? */
static uint64_t stats_n_main_loop_errors = 0;
/** How many times have we returned from the main loop with no events. */
static uint64_t stats_n_main_loop_idle = 0;

/** How often will we honor SIGNEWNYM requests? */
#define MAX_SIGNEWNYM_RATE 10
/** When did we last process a SIGNEWNYM request? */
static time_t time_of_last_signewnym = 0;
/** Is there a signewnym request we're currently waiting to handle? */
static int signewnym_is_pending = 0;
/** Mainloop event for the deferred signewnym call. */
static mainloop_event_t *handle_deferred_signewnym_ev = NULL;
/** How many times have we called newnym? */
static unsigned newnym_epoch = 0;

/** Smartlist of all open connections. */
STATIC smartlist_t *connection_array = NULL;
/** List of connections that have been marked for close and need to be freed
 * and removed from connection_array. */
static smartlist_t *closeable_connection_lst = NULL;
/** List of linked connections that are currently reading data into their
 * inbuf from their partner's outbuf. */
static smartlist_t *active_linked_connection_lst = NULL;
/** Flag: Set to true iff we entered the current libevent main loop via
 * <b>loop_once</b>. If so, there's no need to trigger a loopexit in order
 * to handle linked connections. */
static int called_loop_once = 0;
/** Flag: if true, it's time to shut down, so the main loop should exit as
 * soon as possible.
 */
static int main_loop_should_exit = 0;
/** The return value that the main loop should yield when it exits, if
 * main_loop_should_exit is true.
 */
static int main_loop_exit_value = 0;

/** We set this to 1 when we've opened a circuit, so we can print a log
 * entry to inform the user that Tor is working.  We set it to 0 when
 * we think the fact that we once opened a circuit doesn't mean we can do so
 * any longer (a big time jump happened, when we notice our directory is
 * heinously out-of-date, etc.
 */
static int can_complete_circuits = 0;

/** How often do we check for router descriptors that we should download
 * when we have too little directory info? */
#define GREEDY_DESCRIPTOR_RETRY_INTERVAL (10)
/** How often do we check for router descriptors that we should download
 * when we have enough directory info? */
#define LAZY_DESCRIPTOR_RETRY_INTERVAL (60)

static int conn_close_if_marked(int i);
static void connection_start_reading_from_linked_conn(connection_t *conn);
static int connection_should_read_from_linked_conn(connection_t *conn);
static void conn_read_callback(evutil_socket_t fd, short event, void *_conn);
static void conn_write_callback(evutil_socket_t fd, short event, void *_conn);
static void shutdown_did_not_work_callback(evutil_socket_t fd, short event,
                                           void *arg) ATTR_NORETURN;

/****************************************************************************
 *
 * This section contains accessors and other methods on the connection_array
 * variables (which are global within this file and unavailable outside it).
 *
 ****************************************************************************/

/** Return 1 if we have successfully built a circuit, and nothing has changed
 * to make us think that maybe we can't.
 */
int
have_completed_a_circuit(void)
{
  return can_complete_circuits;
}

/** Note that we have successfully built a circuit, so that reachability
 * testing and introduction points and so on may be attempted. */
void
note_that_we_completed_a_circuit(void)
{
  can_complete_circuits = 1;
}

/** Note that something has happened (like a clock jump, or DisableNetwork) to
 * make us think that maybe we can't complete circuits. */
void
note_that_we_maybe_cant_complete_circuits(void)
{
  can_complete_circuits = 0;
}

/** Add <b>conn</b> to the array of connections that we can poll on.  The
 * connection's socket must be set; the connection starts out
 * non-reading and non-writing.
 */
int
connection_add_impl(connection_t *conn, int is_connecting)
{
  tor_assert(conn);
  tor_assert(SOCKET_OK(conn->s) ||
             conn->linked ||
             (conn->type == CONN_TYPE_AP &&
              TO_EDGE_CONN(conn)->is_dns_request));

  tor_assert(conn->conn_array_index == -1); /* can only connection_add once */
  conn->conn_array_index = smartlist_len(connection_array);
  smartlist_add(connection_array, conn);

  (void) is_connecting;

  if (SOCKET_OK(conn->s) || conn->linked) {
    conn->read_event = tor_event_new(tor_libevent_get_base(),
         conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn);
    conn->write_event = tor_event_new(tor_libevent_get_base(),
         conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn);
    /* XXXX CHECK FOR NULL RETURN! */
  }

  log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.",
            conn_type_to_string(conn->type), (int)conn->s, conn->address,
            smartlist_len(connection_array));

  return 0;
}

/** Tell libevent that we don't care about <b>conn</b> any more. */
void
connection_unregister_events(connection_t *conn)
{
  if (conn->read_event) {
    if (event_del(conn->read_event))
      log_warn(LD_BUG, "Error removing read event for %d", (int)conn->s);
    tor_free(conn->read_event);
  }
  if (conn->write_event) {
    if (event_del(conn->write_event))
      log_warn(LD_BUG, "Error removing write event for %d", (int)conn->s);
    tor_free(conn->write_event);
  }
  if (conn->type == CONN_TYPE_AP_DNS_LISTENER) {
    dnsserv_close_listener(conn);
  }
}

/** Remove the connection from the global list, and remove the
 * corresponding poll entry.  Calling this function will shift the last
 * connection (if any) into the position occupied by conn.
 */
int
connection_remove(connection_t *conn)
{
  int current_index;
  connection_t *tmp;

  tor_assert(conn);

  log_debug(LD_NET,"removing socket %d (type %s), n_conns now %d",
            (int)conn->s, conn_type_to_string(conn->type),
            smartlist_len(connection_array));

  if (conn->type == CONN_TYPE_AP && conn->socket_family == AF_UNIX) {
    log_info(LD_NET, "Closing SOCKS Unix socket connection");
  }

  control_event_conn_bandwidth(conn);

  tor_assert(conn->conn_array_index >= 0);
  current_index = conn->conn_array_index;
  connection_unregister_events(conn); /* This is redundant, but cheap. */
  if (current_index == smartlist_len(connection_array)-1) { /* at the end */
    smartlist_del(connection_array, current_index);
    return 0;
  }

  /* replace this one with the one at the end */
  smartlist_del(connection_array, current_index);
  tmp = smartlist_get(connection_array, current_index);
  tmp->conn_array_index = current_index;

  return 0;
}

/** If <b>conn</b> is an edge conn, remove it from the list
 * of conn's on this circuit. If it's not on an edge,
 * flush and send destroys for all circuits on this conn.
 *
 * Remove it from connection_array (if applicable) and
 * from closeable_connection_list.
 *
 * Then free it.
 */
static void
connection_unlink(connection_t *conn)
{
  connection_about_to_close_connection(conn);
  if (conn->conn_array_index >= 0) {
    connection_remove(conn);
  }
  if (conn->linked_conn) {
    conn->linked_conn->linked_conn = NULL;
    if (! conn->linked_conn->marked_for_close &&
        conn->linked_conn->reading_from_linked_conn)
      connection_start_reading(conn->linked_conn);
    conn->linked_conn = NULL;
  }
  smartlist_remove(closeable_connection_lst, conn);
  smartlist_remove(active_linked_connection_lst, conn);
  if (conn->type == CONN_TYPE_EXIT) {
    assert_connection_edge_not_dns_pending(TO_EDGE_CONN(conn));
  }
  if (conn->type == CONN_TYPE_OR) {
    if (!tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest))
      connection_or_clear_identity(TO_OR_CONN(conn));
    /* connection_unlink() can only get called if the connection
     * was already on the closeable list, and it got there by
     * connection_mark_for_close(), which was called from
     * connection_or_close_normally() or
     * connection_or_close_for_error(), so the channel should
     * already be in CHANNEL_STATE_CLOSING, and then the
     * connection_about_to_close_connection() goes to
     * connection_or_about_to_close(), which calls channel_closed()
     * to notify the channel_t layer, and closed the channel, so
     * nothing more to do here to deal with the channel associated
     * with an orconn.
     */
  }
  connection_free(conn);
}

/** Event that invokes schedule_active_linked_connections_cb. */
static mainloop_event_t *schedule_active_linked_connections_event = NULL;

/**
 * Callback: used to activate read events for all linked connections, so
 * libevent knows to call their read callbacks.  This callback run as a
 * postloop event, so that the events _it_ activates don't happen until
 * Libevent has a chance to check for other events.
 */
static void
schedule_active_linked_connections_cb(mainloop_event_t *event, void *arg)
{
  (void)event;
  (void)arg;

  /* All active linked conns should get their read events activated,
   * so that libevent knows to run their callbacks. */
  SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn,
                    event_active(conn->read_event, EV_READ, 1));

  /* Reactivate the event if we still have connections in the active list.
   *
   * A linked connection doesn't get woken up by I/O but rather artificially
   * by this event callback. It has directory data spooled in it and it is
   * sent incrementally by small chunks unless spool_eagerly is true. For that
   * to happen, we need to induce the activation of the read event so it can
   * be flushed. */
  if (smartlist_len(active_linked_connection_lst)) {
    mainloop_event_activate(schedule_active_linked_connections_event);
  }
}

/** Initialize the global connection list, closeable connection list,
 * and active connection list. */
void
tor_init_connection_lists(void)
{
  if (!connection_array)
    connection_array = smartlist_new();
  if (!closeable_connection_lst)
    closeable_connection_lst = smartlist_new();
  if (!active_linked_connection_lst)
    active_linked_connection_lst = smartlist_new();
}

/** Schedule <b>conn</b> to be closed. **/
void
add_connection_to_closeable_list(connection_t *conn)
{
  tor_assert(!smartlist_contains(closeable_connection_lst, conn));
  tor_assert(conn->marked_for_close);
  assert_connection_ok(conn, time(NULL));
  smartlist_add(closeable_connection_lst, conn);
  mainloop_schedule_postloop_cleanup();
}

/** Return 1 if conn is on the closeable list, else return 0. */
int
connection_is_on_closeable_list(connection_t *conn)
{
  return smartlist_contains(closeable_connection_lst, conn);
}

/** Return true iff conn is in the current poll array. */
int
connection_in_array(connection_t *conn)
{
  return smartlist_contains(connection_array, conn);
}

/** Set <b>*array</b> to an array of all connections. <b>*array</b> must not
 * be modified.
 */
MOCK_IMPL(smartlist_t *,
get_connection_array, (void))
{
  if (!connection_array)
    connection_array = smartlist_new();
  return connection_array;
}

/**
 * Return the amount of network traffic read, in bytes, over the life of this
 * process.
 */
MOCK_IMPL(uint64_t,
get_bytes_read,(void))
{
  return stats_n_bytes_read;
}

/**
 * Return the amount of network traffic read, in bytes, over the life of this
 * process.
 */
MOCK_IMPL(uint64_t,
get_bytes_written,(void))
{
  return stats_n_bytes_written;
}

/**
 * Increment the amount of network traffic read and written, over the life of
 * this process.
 */
void
stats_increment_bytes_read_and_written(uint64_t r, uint64_t w)
{
  stats_n_bytes_read += r;
  stats_n_bytes_written += w;
}

/** Set the event mask on <b>conn</b> to <b>events</b>.  (The event
 * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT)
 */
void
connection_watch_events(connection_t *conn, watchable_events_t events)
{
  if (events & READ_EVENT)
    connection_start_reading(conn);
  else
    connection_stop_reading(conn);

  if (events & WRITE_EVENT)
    connection_start_writing(conn);
  else
    connection_stop_writing(conn);
}

/** Return true iff <b>conn</b> is listening for read events. */
int
connection_is_reading(connection_t *conn)
{
  tor_assert(conn);

  return conn->reading_from_linked_conn ||
    (conn->read_event && event_pending(conn->read_event, EV_READ, NULL));
}

/** Reset our main loop counters. */
void
reset_main_loop_counters(void)
{
  stats_n_main_loop_successes = 0;
  stats_n_main_loop_errors = 0;
  stats_n_main_loop_idle = 0;
}

/** Increment the main loop success counter. */
static void
increment_main_loop_success_count(void)
{
  ++stats_n_main_loop_successes;
}

/** Get the main loop success counter. */
uint64_t
get_main_loop_success_count(void)
{
  return stats_n_main_loop_successes;
}

/** Increment the main loop error counter. */
static void
increment_main_loop_error_count(void)
{
  ++stats_n_main_loop_errors;
}

/** Get the main loop error counter. */
uint64_t
get_main_loop_error_count(void)
{
  return stats_n_main_loop_errors;
}

/** Increment the main loop idle counter. */
static void
increment_main_loop_idle_count(void)
{
  ++stats_n_main_loop_idle;
}

/** Get the main loop idle counter. */
uint64_t
get_main_loop_idle_count(void)
{
  return stats_n_main_loop_idle;
}

/** Check whether <b>conn</b> is correct in having (or not having) a
 * read/write event (passed in <b>ev</b>). On success, return 0. On failure,
 * log a warning and return -1. */
static int
connection_check_event(connection_t *conn, struct event *ev)
{
  int bad;

  if (conn->type == CONN_TYPE_AP && TO_EDGE_CONN(conn)->is_dns_request) {
    /* DNS requests which we launch through the dnsserv.c module do not have
     * any underlying socket or any underlying linked connection, so they
     * shouldn't have any attached events either.
     */
    bad = ev != NULL;
  } else {
    /* Everything else should have an underlying socket, or a linked
     * connection (which is also tracked with a read_event/write_event pair).
     */
    bad = ev == NULL;
  }

  if (bad) {
    log_warn(LD_BUG, "Event missing on connection %p [%s;%s]. "
             "socket=%d. linked=%d. "
             "is_dns_request=%d. Marked_for_close=%s:%d",
             conn,
             conn_type_to_string(conn->type),
             conn_state_to_string(conn->type, conn->state),
             (int)conn->s, (int)conn->linked,
             (conn->type == CONN_TYPE_AP &&
                               TO_EDGE_CONN(conn)->is_dns_request),
             conn->marked_for_close_file ? conn->marked_for_close_file : "-",
             conn->marked_for_close
             );
    log_backtrace(LOG_WARN, LD_BUG, "Backtrace attached.");
    return -1;
  }
  return 0;
}

/** Tell the main loop to stop notifying <b>conn</b> of any read events. */
MOCK_IMPL(void,
connection_stop_reading,(connection_t *conn))
{
  tor_assert(conn);

  if (connection_check_event(conn, conn->read_event) < 0) {
    return;
  }

  if (conn->linked) {
    conn->reading_from_linked_conn = 0;
    connection_stop_reading_from_linked_conn(conn);
  } else {
    if (event_del(conn->read_event))
      log_warn(LD_NET, "Error from libevent setting read event state for %d "
               "to unwatched: %s",
               (int)conn->s,
               tor_socket_strerror(tor_socket_errno(conn->s)));
  }
}

/** Tell the main loop to start notifying <b>conn</b> of any read events. */
MOCK_IMPL(void,
connection_start_reading,(connection_t *conn))
{
  tor_assert(conn);

  if (connection_check_event(conn, conn->read_event) < 0) {
    return;
  }

  if (conn->linked) {
    conn->reading_from_linked_conn = 1;
    if (connection_should_read_from_linked_conn(conn))
      connection_start_reading_from_linked_conn(conn);
  } else {
    if (event_add(conn->read_event, NULL))
      log_warn(LD_NET, "Error from libevent setting read event state for %d "
               "to watched: %s",
               (int)conn->s,
               tor_socket_strerror(tor_socket_errno(conn->s)));
  }
}

/** Return true iff <b>conn</b> is listening for write events. */
int
connection_is_writing(connection_t *conn)
{
  tor_assert(conn);

  return conn->writing_to_linked_conn ||
    (conn->write_event && event_pending(conn->write_event, EV_WRITE, NULL));
}

/** Tell the main loop to stop notifying <b>conn</b> of any write events. */
MOCK_IMPL(void,
connection_stop_writing,(connection_t *conn))
{
  tor_assert(conn);

  if (connection_check_event(conn, conn->write_event) < 0) {
    return;
  }

  if (conn->linked) {
    conn->writing_to_linked_conn = 0;
    if (conn->linked_conn)
      connection_stop_reading_from_linked_conn(conn->linked_conn);
  } else {
    if (event_del(conn->write_event))
      log_warn(LD_NET, "Error from libevent setting write event state for %d "
               "to unwatched: %s",
               (int)conn->s,
               tor_socket_strerror(tor_socket_errno(conn->s)));
  }
}

/** Tell the main loop to start notifying <b>conn</b> of any write events. */
MOCK_IMPL(void,
connection_start_writing,(connection_t *conn))
{
  tor_assert(conn);

  if (connection_check_event(conn, conn->write_event) < 0) {
    return;
  }

  if (conn->linked) {
    conn->writing_to_linked_conn = 1;
    if (conn->linked_conn &&
        connection_should_read_from_linked_conn(conn->linked_conn))
      connection_start_reading_from_linked_conn(conn->linked_conn);
  } else {
    if (event_add(conn->write_event, NULL))
      log_warn(LD_NET, "Error from libevent setting write event state for %d "
               "to watched: %s",
               (int)conn->s,
               tor_socket_strerror(tor_socket_errno(conn->s)));
  }
}

/** Return true iff <b>conn</b> is linked conn, and reading from the conn
 * linked to it would be good and feasible.  (Reading is "feasible" if the
 * other conn exists and has data in its outbuf, and is "good" if we have our
 * reading_from_linked_conn flag set and the other conn has its
 * writing_to_linked_conn flag set.)*/
static int
connection_should_read_from_linked_conn(connection_t *conn)
{
  if (conn->linked && conn->reading_from_linked_conn) {
    if (! conn->linked_conn ||
        (conn->linked_conn->writing_to_linked_conn &&
         buf_datalen(conn->linked_conn->outbuf)))
      return 1;
  }
  return 0;
}

/** Event to run 'shutdown did not work callback'. */
static struct event *shutdown_did_not_work_event = NULL;

/** Failsafe measure that should never actually be necessary: If
 * tor_shutdown_event_loop_and_exit() somehow doesn't successfully exit the
 * event loop, then this callback will kill Tor with an assertion failure
 * seconds later
 */
static void
shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg)
{
  // LCOV_EXCL_START
  (void) fd;
  (void) event;
  (void) arg;
  tor_assert_unreached();
  // LCOV_EXCL_STOP
}

#ifdef ENABLE_RESTART_DEBUGGING
static struct event *tor_shutdown_event_loop_for_restart_event = NULL;
static void
tor_shutdown_event_loop_for_restart_cb(
                      evutil_socket_t fd, short event, void *arg)
{
  (void)fd;
  (void)event;
  (void)arg;
  tor_event_free(tor_shutdown_event_loop_for_restart_event);
  tor_shutdown_event_loop_and_exit(0);
}
#endif

/**
 * After finishing the current callback (if any), shut down the main loop,
 * clean up the process, and exit with <b>exitcode</b>.
 */
void
tor_shutdown_event_loop_and_exit(int exitcode)
{
  if (main_loop_should_exit)
    return; /* Ignore multiple calls to this function. */

  main_loop_should_exit = 1;
  main_loop_exit_value = exitcode;

  /* Die with an assertion failure in ten seconds, if for some reason we don't
   * exit normally. */
  /* XXXX We should consider this code if it's never used. */
  struct timeval ten_seconds = { 10, 0 };
  shutdown_did_not_work_event = tor_evtimer_new(
                  tor_libevent_get_base(),
                  shutdown_did_not_work_callback, NULL);
  event_add(shutdown_did_not_work_event, &ten_seconds);

  /* Unlike exit_loop_after_delay(), exit_loop_after_callback
   * prevents other callbacks from running. */
  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
}

/** Return true iff tor_shutdown_event_loop_and_exit() has been called. */
int
tor_event_loop_shutdown_is_pending(void)
{
  return main_loop_should_exit;
}

/** Helper: Tell the main loop to begin reading bytes into <b>conn</b> from
 * its linked connection, if it is not doing so already.  Called by
 * connection_start_reading and connection_start_writing as appropriate. */
static void
connection_start_reading_from_linked_conn(connection_t *conn)
{
  tor_assert(conn);
  tor_assert(conn->linked == 1);

  if (!conn->active_on_link) {
    conn->active_on_link = 1;
    smartlist_add(active_linked_connection_lst, conn);
    mainloop_event_activate(schedule_active_linked_connections_event);
  } else {
    tor_assert(smartlist_contains(active_linked_connection_lst, conn));
  }
}

/** Tell the main loop to stop reading bytes into <b>conn</b> from its linked
 * connection, if is currently doing so.  Called by connection_stop_reading,
 * connection_stop_writing, and connection_read. */
void
connection_stop_reading_from_linked_conn(connection_t *conn)
{
  tor_assert(conn);
  tor_assert(conn->linked == 1);

  if (conn->active_on_link) {
    conn->active_on_link = 0;
    /* FFFF We could keep an index here so we can smartlist_del
     * cleanly.  On the other hand, this doesn't show up on profiles,
     * so let's leave it alone for now. */
    smartlist_remove(active_linked_connection_lst, conn);
  } else {
    tor_assert(!smartlist_contains(active_linked_connection_lst, conn));
  }
}

/** Close all connections that have been scheduled to get closed. */
STATIC void
close_closeable_connections(void)
{
  int i;
  for (i = 0; i < smartlist_len(closeable_connection_lst); ) {
    connection_t *conn = smartlist_get(closeable_connection_lst, i);
    if (conn->conn_array_index < 0) {
      connection_unlink(conn); /* blow it away right now */
    } else {
      if (!conn_close_if_marked(conn->conn_array_index))
        ++i;
    }
  }
}

/** Count moribund connections for the OOS handler */
MOCK_IMPL(int,
connection_count_moribund, (void))
{
  int moribund = 0;

  /*
   * Count things we'll try to kill when close_closeable_connections()
   * runs next.
   */
  SMARTLIST_FOREACH_BEGIN(closeable_connection_lst, connection_t *, conn) {
    if (SOCKET_OK(conn->s) && connection_is_moribund(conn)) ++moribund;
  } SMARTLIST_FOREACH_END(conn);

  return moribund;
}

/** Libevent callback: this gets invoked when (connection_t*)<b>conn</b> has
 * some data to read. */
static void
conn_read_callback(evutil_socket_t fd, short event, void *_conn)
{
  connection_t *conn = _conn;
  (void)fd;
  (void)event;

  log_debug(LD_NET,"socket %d wants to read.",(int)conn->s);

  /* assert_connection_ok(conn, time(NULL)); */

  if (connection_handle_read(conn) < 0) {
    if (!conn->marked_for_close) {
#ifndef _WIN32
      log_warn(LD_BUG,"Unhandled error on read for %s connection "
               "(fd %d); removing",
               conn_type_to_string(conn->type), (int)conn->s);
      tor_fragile_assert();
#endif /* !defined(_WIN32) */
      if (CONN_IS_EDGE(conn))
        connection_edge_end_errno(TO_EDGE_CONN(conn));
      connection_mark_for_close(conn);
    }
  }
  assert_connection_ok(conn, time(NULL));

  if (smartlist_len(closeable_connection_lst))
    close_closeable_connections();
}

/** Libevent callback: this gets invoked when (connection_t*)<b>conn</b> has
 * some data to write. */
static void
conn_write_callback(evutil_socket_t fd, short events, void *_conn)
{
  connection_t *conn = _conn;
  (void)fd;
  (void)events;

  LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "socket %d wants to write.",
                     (int)conn->s));

  /* assert_connection_ok(conn, time(NULL)); */

  if (connection_handle_write(conn, 0) < 0) {
    if (!conn->marked_for_close) {
      /* this connection is broken. remove it. */
      log_fn(LOG_WARN,LD_BUG,
             "unhandled error on write for %s connection (fd %d); removing",
             conn_type_to_string(conn->type), (int)conn->s);
      tor_fragile_assert();
      if (CONN_IS_EDGE(conn)) {
        /* otherwise we cry wolf about duplicate close */
        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
        if (!edge_conn->end_reason)
          edge_conn->end_reason = END_STREAM_REASON_INTERNAL;
        edge_conn->edge_has_sent_end = 1;
      }
      connection_close_immediate(conn); /* So we don't try to flush. */
      connection_mark_for_close(conn);
    }
  }
  assert_connection_ok(conn, time(NULL));

  if (smartlist_len(closeable_connection_lst))
    close_closeable_connections();
}

/** If the connection at connection_array[i] is marked for close, then:
 *    - If it has data that it wants to flush, try to flush it.
 *    - If it _still_ has data to flush, and conn->hold_open_until_flushed is
 *      true, then leave the connection open and return.
 *    - Otherwise, remove the connection from connection_array and from
 *      all other lists, close it, and free it.
 * Returns 1 if the connection was closed, 0 otherwise.
 */
static int
conn_close_if_marked(int i)
{
  connection_t *conn;
  int retval;
  time_t now;

  conn = smartlist_get(connection_array, i);
  if (!conn->marked_for_close)
    return 0; /* nothing to see here, move along */
  now = time(NULL);
  assert_connection_ok(conn, now);
  /* assert_all_pending_dns_resolves_ok(); */

  log_debug(LD_NET,"Cleaning up connection (fd "TOR_SOCKET_T_FORMAT").",
            conn->s);

  /* If the connection we are about to close was trying to connect to
  a proxy server and failed, the client won't be able to use that
  proxy. We should warn the user about this. */
  if (conn->proxy_state == PROXY_INFANT)
    log_failed_proxy_connection(conn);

  if ((SOCKET_OK(conn->s) || conn->linked_conn) &&
      connection_wants_to_flush(conn)) {
    /* s == -1 means it's an incomplete edge connection, or that the socket
     * has already been closed as unflushable. */
    ssize_t sz = connection_bucket_write_limit(conn, now);
    if (!conn->hold_open_until_flushed)
      log_info(LD_NET,
               "Conn (addr %s, fd %d, type %s, state %d) marked, but wants "
               "to flush %d bytes. (Marked at %s:%d)",
               escaped_safe_str_client(conn->address),
               (int)conn->s, conn_type_to_string(conn->type), conn->state,
               (int)conn->outbuf_flushlen,
                conn->marked_for_close_file, conn->marked_for_close);
    if (conn->linked_conn) {
      retval = buf_move_to_buf(conn->linked_conn->inbuf, conn->outbuf,
                               &conn->outbuf_flushlen);
      if (retval >= 0) {
        /* The linked conn will notice that it has data when it notices that
         * we're gone. */
        connection_start_reading_from_linked_conn(conn->linked_conn);
      }
      log_debug(LD_GENERAL, "Flushed last %d bytes from a linked conn; "
               "%d left; flushlen %d; wants-to-flush==%d", retval,
                (int)connection_get_outbuf_len(conn),
                (int)conn->outbuf_flushlen,
                connection_wants_to_flush(conn));
    } else if (connection_speaks_cells(conn)) {
      if (conn->state == OR_CONN_STATE_OPEN) {
        retval = buf_flush_to_tls(conn->outbuf, TO_OR_CONN(conn)->tls, sz,
                               &conn->outbuf_flushlen);
      } else
        retval = -1; /* never flush non-open broken tls connections */
    } else {