Make Tor use Niels Provos's libevent instead of it's current

poll-but-sometimes-select mess.  This will let us use faster async cores
(like epoll, kpoll, and /dev/poll), and hopefully work better on Windows
too.

There are some fairly nasty changes to main.c here; this will almost
certainly break something.  But hey, that's what alphas are for.


svn:r3341

configure.in

@@ -32,6 +32,7 @@ AC_ARG_WITH(ssl-dir,
 
 AC_SEARCH_LIBS(socket, [socket])
 AC_SEARCH_LIBS(gethostbyname, [nsl])
+AC_SEARCH_LIBS(event_loop, [event], , AC_MSG_ERROR(Libevent library not found. Tor requires libevent to build.))
 
 saved_LIBS="$LIBS"
 saved_LDFLAGS="$LDFLAGS"
@@ -139,7 +140,9 @@ AC_SYS_LARGEFILE
 
 dnl The warning message here is no longer strictly accurate.
 
-AC_CHECK_HEADERS(unistd.h string.h signal.h netdb.h ctype.h poll.h sys/stat.h sys/poll.h sys/types.h fcntl.h sys/fcntl.h sys/ioctl.h sys/socket.h sys/time.h netinet/in.h arpa/inet.h errno.h assert.h time.h pwd.h grp.h zlib.h, , AC_MSG_WARN(some headers were not found, compilation may fail))
+AC_CHECK_HEADERS(unistd.h string.h signal.h netdb.h ctype.h sys/stat.h sys/types.h fcntl.h sys/fcntl.h sys/ioctl.h sys/socket.h sys/time.h netinet/in.h arpa/inet.h errno.h assert.h time.h pwd.h grp.h zlib.h, , AC_MSG_WARN(some headers were not found, compilation may fail))
+
+AC_CHECK_HEADERS(event.h, , AC_MSG_ERROR(Libevent header (event.h) not found. Tor requires libevent to build.))
 
 dnl These headers are not essential
 

doc/TODO

@@ -70,7 +70,7 @@ Tier one:
 
    - Windows
 N    - Make millisecond accuracy work on win32
-     - Switch to WSA*Event code as a better poll replacement.  Or maybe just
+     X Switch to WSA*Event code as a better poll replacement.  Or maybe just
        do libevent?
 
    - Code cleanup
@@ -101,10 +101,24 @@ Tier two:
      - Limit number of circuits that we preemptively generate based on past
        behavior; use same limits in circuit_expire_old_circuits().
      - Write limiting; configurable token buckets.
-     - Switch to libevent?  Evaluate it first.
      - Make it harder to circumvent bandwidth caps: look at number of bytes
        sent across sockets, not number sent inside TLS stream.
 
+     . Switch to libevent
+       o Evaluate libevent
+       o Convert socket handling
+       o Convert signal handling
+       o Convert timers
+       o Update configure.in
+       o Remove fakepoll
+       - Hold-open-until-flushed now works by accident; it should work by
+         design.
+       - The logic for reading from TLS sockets is likely to overrun the
+         bandwidth buckets under heavy load.  (Really, the logic was
+         never right in the first place.)  Also, we should audit all users
+         of get_pending_bytes().
+       - Make sure it works on more platforms.
+       - Find a way to make sure we have libevent 1.0 or later.
 
    - QOI
      - Let more config options (e.g. ORPort) change dynamically.

src/common/Makefile.am

@@ -3,7 +3,7 @@ noinst_LIBRARIES = libor.a libor-crypto.a
 
 #CFLAGS  = -Wall -Wpointer-arith -O2
 
-libor_a_SOURCES = log.c fakepoll.c util.c compat.c container.c
+libor_a_SOURCES = log.c util.c compat.c container.c
 libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c
 
-noinst_HEADERS = log.h crypto.h fakepoll.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h
+noinst_HEADERS = log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h

src/common/compat.c

@@ -8,7 +8,6 @@ const char compat_c_id[] = "$Id$";
 #define _GNU_SOURCE
 
 #include "orconfig.h"
-#include "fakepoll.h"
 #include "compat.h"
 
 #ifdef MS_WINDOWS

src/common/compat.h

@@ -115,6 +115,10 @@ int replace_file(const char *from, const char *to);
 #define tor_close_socket(s) close(s)
 #endif
 
+/* Now that we use libevent, all real sockets are safe for polling ... or
+ * if they aren't, libevent will help us. */
+#define SOCKET_IS_POLLABLE(fd) ((fd)>=0)
+
 struct in_addr;
 int tor_inet_aton(const char *cp, struct in_addr *addr);
 int tor_lookup_hostname(const char *name, uint32_t *addr);

src/common/fakepoll.c

-/* Copyright 2002,2003 Nick Mathewson, Roger Dingledine */
-/* See LICENSE for licensing information */
-/* $Id$ */
-const char fakepoll_c_id[] = "$Id$";
-
-/**
- * \file fakepoll.c
- *
- * \brief On systems where poll() doesn't exist, fake it with select().
- **/
-
-#include "orconfig.h"
-#include "fakepoll.h"
-
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-
-#include <assert.h>
-#include <stdlib.h>
-#include "util.h"
-#include "log.h"
-
-#ifndef USE_FAKE_POLL
-int
-tor_poll(struct pollfd *ufds, unsigned int nfds, int timeout)
-{
-        unsigned int i;
-        for (i=0;i<nfds;++i) {
-                tor_assert(ufds[i].fd >= 0);
-        }
-        return poll(ufds,nfds,timeout);
-}
-#else
-int
-tor_poll(struct pollfd *ufds, unsigned int nfds, int timeout)
-{
-        unsigned int idx;
-        int maxfd, fd;
-        int r;
-#ifdef MS_WINDOWS
-        int any_fds_set = 0;
-#endif
-        fd_set readfds, writefds, exceptfds;
-#ifdef USING_FAKE_TIMEVAL
-#undef timeval
-#undef tv_sec
-#undef tv_usec
-#endif
-        struct timeval _timeout;
-        _timeout.tv_sec = timeout/1000;
-        _timeout.tv_usec = (timeout%1000)*1000;
-        FD_ZERO(&readfds);
-        FD_ZERO(&writefds);
-        FD_ZERO(&exceptfds);
-
-        maxfd = -1;
-        for (idx = 0; idx < nfds; ++idx) {
-                ufds[idx].revents = 0;
-                fd = ufds[idx].fd;
-                tor_assert(SOCKET_IS_POLLABLE(fd));
-                if (fd > maxfd) {
-                  maxfd = fd;
-#ifdef MS_WINDOWS
-                  any_fds_set = 1;
-#endif
-                }
-                if (ufds[idx].events & POLLIN)
-                        FD_SET(fd, &readfds);
-                if (ufds[idx].events & POLLOUT)
-                        FD_SET(fd, &writefds);
-                FD_SET(fd, &exceptfds);
-        }
-#ifdef MS_WINDOWS
-        if (!any_fds_set) {
-                Sleep(timeout);
-                return 0;
-        }
-#endif
-        r = select(maxfd+1, &readfds, &writefds, &exceptfds,
-                   timeout == -1 ? NULL : &_timeout);
-        if (r <= 0)
-                return r;
-        r = 0;
-        for (idx = 0; idx < nfds; ++idx) {
-                fd = ufds[idx].fd;
-                if (FD_ISSET(fd, &readfds))
-                        ufds[idx].revents |= POLLIN;
-                if (FD_ISSET(fd, &writefds))
-                        ufds[idx].revents |= POLLOUT;
-                if (FD_ISSET(fd, &exceptfds))
-                        ufds[idx].revents |= POLLERR;
-                if (ufds[idx].revents)
-                        ++r;
-        }
-        return r;
-}
-#endif
-

src/common/fakepoll.h

-/* Copyright 2002,2003 Nick Mathewson, Roger Dingledine. */
-/* See LICENSE for licensing information */
-/* $Id$ */
-
-#ifndef __FAKEPOLL_H
-#define __FAKEPOLL_H
-#define FAKEPOLL_H_ID "$Id$"
-
-/**
- * \file fakepoll.h
- * \brief Headers for fakepoll.c
- */
-
-#include "orconfig.h"
-
-#define POLL_NO_WARN
-
-#if defined(HAVE_POLL_H)
-#include <poll.h>
-#elif defined(HAVE_SYS_POLL_H)
-#include <sys/poll.h>
-#endif
-
-/* If _POLL_EMUL_H_ is defined, then poll is just a just a thin wrapper around
- * select.  On Mac OS 10.3, this wrapper is kinda flaky, and we should
- * use our own.
- */
-#if !(defined(HAVE_POLL_H)||defined(HAVE_SYS_POLL_H))&&!defined(_POLL_EMUL_H_)
-#define USE_FAKE_POLL
-#endif
-
-#if defined USE_FAKE_POLL && !defined(_POLL_EMUL_H_)
-struct pollfd {
-  int fd;
-  short events;
-  short revents;
-};
-
-#define POLLIN   0x0001
-#define POLLPRI  0x0002
-#define POLLOUT  0x0004
-#define POLLERR  0x0008
-#define POLLHUP  0x0010
-#define POLLNVAL 0x0020
-#endif
-
-#ifdef MS_WINDOWS
-#define MAXCONNECTIONS 10000 /* XXXX copied from or.h */
-/* This trick makes winsock resize fd_set, which defaults to the insanely low
- * 64. */
-#define FD_SETSIZE MAXCONNECTIONS
-/* XXXX But Windows FD_SET and FD_CLR are tremendously ugly, and linear in
- *   the total number of sockets set! Perhaps we should eventually use
- *   WSAEventSelect and WSAWaitForMultipleEvents instead of select? */
-#endif
-
-#if defined(MS_WINDOWS) || ! defined(USE_FAKE_POLL)
-/* If we're using poll, we can poll as many sockets as we want.
- * If we're on Windows, having too many sockets is harmless, since
- * select stupidly uses an array of sockets rather than a bitfield. */
-#define SOCKET_IS_POLLABLE(fd) ((fd) >= 0)
-#else
-/* If we're using a real Posix select, then in order to be pollable, a socket
- * must
- *   a) be valid (>= 0)
- *   b) be < FD_SETSIZE.
- */
-#define SOCKET_IS_POLLABLE(fd) ((fd) >= 0 && (fd) < FD_SETSIZE)
-#endif
-
-int tor_poll(struct pollfd *ufds, unsigned int nfds, int timeout);
-
-#endif

src/or/config.c

@@ -2435,7 +2435,6 @@ print_cvs_version(void)
   extern const char compat_c_id[];
   extern const char container_c_id[];
   extern const char crypto_c_id[];
-  extern const char fakepoll_c_id[];
   extern const char log_c_id[];
   extern const char torgzip_c_id[];
   extern const char tortls_c_id[];
@@ -2472,7 +2471,6 @@ print_cvs_version(void)
   puts(COMPAT_H_ID);
   puts(CONTAINER_H_ID);
   puts(CRYPTO_H_ID);
-  puts(FAKEPOLL_H_ID);
   puts(LOG_H_ID);
   puts(TORGZIP_H_ID);
   puts(TORINT_H_ID);
@@ -2482,7 +2480,6 @@ print_cvs_version(void)
   puts(compat_c_id);
   puts(container_c_id);
   puts(crypto_c_id);
-  puts(fakepoll_c_id);
   puts(log_c_id);
   puts(torgzip_c_id);
   puts(tortls_c_id);

src/or/connection.c

@@ -167,6 +167,14 @@ void connection_free(connection_t *conn) {
     log_fn(LOG_INFO,"closing fd %d.",conn->s);
     tor_close_socket(conn->s);
   }
+  if (conn->read_event) {
+    event_del(conn->read_event);
+    tor_free(conn->read_event);
+  }
+  if (conn->write_event) {
+    event_del(conn->write_event);
+    tor_free(conn->write_event);
+  }
   memset(conn, 0xAA, sizeof(connection_t)); /* poison memory */
   tor_free(conn);
 }
@@ -300,6 +308,7 @@ _connection_mark_for_close(connection_t *conn)
   }
 
   conn->marked_for_close = 1;
+  add_connection_to_closeable_list(conn);
 
   /* in case we're going to be held-open-til-flushed, reset
    * the number of seconds since last successful write, so
@@ -904,6 +913,7 @@ static int connection_read_to_buf(connection_t *conn, int *max_to_read) {
   }
 
   if (connection_speaks_cells(conn) && conn->state != OR_CONN_STATE_CONNECTING) {
+    int pending;
     if (conn->state == OR_CONN_STATE_HANDSHAKING) {
       /* continue handshaking even if global token bucket is empty */
       return connection_tls_continue_handshake(conn);
@@ -931,7 +941,22 @@ static int connection_read_to_buf(connection_t *conn, int *max_to_read) {
       case TOR_TLS_DONE: /* no data read, so nothing to process */
         result = 0;
         break; /* so we call bucket_decrement below */
+      default:
+        break;
+    }
+    pending = tor_tls_get_pending_bytes(conn->tls);
+    if (pending) {
+      /* XXXX If we have any pending bytes, read them now.  This *can*
+       * take us over our read alotment, but really we shouldn't be
+       * believing that SSL bytes are the same as TCP bytes anyway. */
+      int r2 = read_to_buf_tls(conn->tls, pending, conn->inbuf);
+      if (r2<0) {
+        log_fn(LOG_WARN, "Bug: apparently, reading pending bytes can fail.");
+      } else {
+        result += r2;
+      }
     }
+
   } else {
     result = read_to_buf(conn->s, at_most, conn->inbuf,
                          &conn->inbuf_reached_eof);

src/or/connection_edge.c

@@ -203,7 +203,7 @@ int connection_edge_finished_connecting(connection_t *conn)
          conn->address,conn->port);
 
   conn->state = EXIT_CONN_STATE_OPEN;
-  connection_watch_events(conn, POLLIN); /* stop writing, continue reading */
+  connection_watch_events(conn, EV_READ); /* stop writing, continue reading */
   if (connection_wants_to_flush(conn)) /* in case there are any queued relay cells */
     connection_start_writing(conn);
   /* deliver a 'connected' relay cell back through the circuit. */
@@ -949,7 +949,7 @@ connection_exit_connect(connection_t *conn) {
     case 0:
       conn->state = EXIT_CONN_STATE_CONNECTING;
 
-      connection_watch_events(conn, POLLOUT | POLLIN | POLLERR);
+      connection_watch_events(conn, EV_WRITE | EV_READ);
       /* writable indicates finish, readable indicates broken link,
          error indicates broken link in windowsland. */
       return;
@@ -961,7 +961,7 @@ connection_exit_connect(connection_t *conn) {
     log_fn(LOG_WARN,"Bug: newly connected conn had data waiting!");
 //    connection_start_writing(conn);
   }
-  connection_watch_events(conn, POLLIN);
+  connection_watch_events(conn, EV_READ);
 
   /* also, deliver a 'connected' cell back through the circuit. */
   if (connection_edge_is_rendezvous_stream(conn)) { /* rendezvous stream */

src/or/connection_or.c

@@ -238,7 +238,7 @@ connection_t *connection_or_connect(uint32_t addr, uint16_t port,
       connection_free(conn);
       return NULL;
     case 0:
-      connection_watch_events(conn, POLLIN | POLLOUT | POLLERR);
+      connection_watch_events(conn, EV_READ | EV_WRITE);
       /* writable indicates finish, readable indicates broken link,
          error indicates broken link on windows */
       return conn;
@@ -342,7 +342,7 @@ connection_tls_finish_handshake(connection_t *conn) {
   or_options_t *options = get_options();
 
   conn->state = OR_CONN_STATE_OPEN;
-  connection_watch_events(conn, POLLIN);
+  connection_watch_events(conn, EV_READ);
   log_fn(LOG_DEBUG,"tls handshake done. verifying.");
   if (! tor_tls_peer_has_cert(conn->tls)) { /* It's an old OP. */
     if (server_mode(options)) { /* I'm an OR; good. */

src/or/directory.c

@@ -321,7 +321,7 @@ directory_initiate_command(const char *address, uint32_t addr,
         /* queue the command on the outbuf */
         directory_send_command(conn, platform, purpose, resource,
                                payload, payload_len);
-        connection_watch_events(conn, POLLIN | POLLOUT | POLLERR);
+        connection_watch_events(conn, EV_READ | EV_WRITE);
         /* writable indicates finish, readable indicates broken link,
            error indicates broken link in windowsland. */
     }
@@ -342,7 +342,7 @@ directory_initiate_command(const char *address, uint32_t addr,
     /* queue the command on the outbuf */
     directory_send_command(conn, platform, purpose, resource,
                            payload, payload_len);
-    connection_watch_events(conn, POLLIN | POLLOUT | POLLERR);
+    connection_watch_events(conn, EV_READ | EV_WRITE);
   }
 }
 

src/or/or.h

@@ -40,7 +40,6 @@
 #include <ctype.h>
 #endif
 #include "../common/torint.h"
-#include "../common/fakepoll.h"
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
@@ -97,6 +96,11 @@
 #ifdef HAVE_TIME_H
 #include <time.h>
 #endif
+#ifdef HAVE_EVENT_H
+#include <event.h>
+#else
+#error "Tor requires libevent to build."
+#endif
 
 #ifdef MS_WINDOWS
 #if (_MSC_VER <= 1300)
@@ -494,7 +498,9 @@ struct connection_t {
                            * the bandwidth throttler allows reads?
                            */
   int s; /**< Our socket; -1 if this connection is closed. */
-  int poll_index; /**< Index of this conn into the poll_array. */
+  int poll_index; /* XXXX rename. */
+  struct event *read_event; /**< libevent event structure. */
+  struct event *write_event; /**< libevent event structure. */
   int marked_for_close; /**< Boolean: should we close this conn on the next
                          * iteration of the main loop?
                          */
@@ -1380,6 +1386,7 @@ void consider_hibernation(time_t now);
 int connection_add(connection_t *conn);
 int connection_remove(connection_t *conn);
 int connection_in_array(connection_t *conn);
+void add_connection_to_closeable_list(connection_t *conn);
 
 void get_connection_array(connection_t ***array, int *n);