Commit cf2ac8e2 authored by Nick Mathewson's avatar Nick Mathewson 🦀
Browse files

Merge remote-tracking branch 'public/feature11791'

parents 6f7a8f84 734ba5cb

changes/bug11791

0 → 100644
+4 −0
Original line number Diff line number Diff line 
  o Minor features (directory, memory usage):

    - When we have recently been under memory pressure (over 3/4 of

      MaxMemInQueues is allocated), then allocate smaller zlib objects for

      small requests. Closes ticket 11791.

src/common/torgzip.c

+39 −11
Original line number Diff line number Diff line
@@ -92,10 +92,27 @@ tor_zlib_get_header_version_str(void) 


/** Return the 'bits' value to tell zlib to use <b>method</b>.*/

static INLINE int

method_bits(compress_method_t method)

method_bits(compress_method_t method, zlib_compression_level_t level)

{

  /* Bits+16 means "use gzip" in zlib >= 1.2 */

  return method == GZIP_METHOD ? 15+16 : 15;

  const int flag = method == GZIP_METHOD ? 16 : 0;

  switch (level) {

    default:

    case HIGH_COMPRESSION: return flag + 15;

    case MEDIUM_COMPRESSION: return flag + 13;

    case LOW_COMPRESSION: return flag + 11;

  }

}



static INLINE int

get_memlevel(zlib_compression_level_t level)

{

  switch (level) {

    default:

    case HIGH_COMPRESSION: return 8;

    case MEDIUM_COMPRESSION: return 7;

    case LOW_COMPRESSION: return 6;

  }

}



/** @{ */
@@ -162,8 +179,9 @@ tor_gzip_compress(char **out, size_t *out_len, 
  stream->avail_in = (unsigned int)in_len;



  if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,

                   method_bits(method),

                   8, Z_DEFAULT_STRATEGY) != Z_OK) {

                   method_bits(method, HIGH_COMPRESSION),

                   get_memlevel(HIGH_COMPRESSION),

                   Z_DEFAULT_STRATEGY) != Z_OK) {

    log_warn(LD_GENERAL, "Error from deflateInit2: %s",

             stream->msg?stream->msg:"<no message>");

    goto err;
@@ -289,7 +307,7 @@ tor_gzip_uncompress(char **out, size_t *out_len, 
  stream->avail_in = (unsigned int)in_len;



  if (inflateInit2(stream,

                   method_bits(method)) != Z_OK) {

                   method_bits(method, HIGH_COMPRESSION)) != Z_OK) {

    log_warn(LD_GENERAL, "Error from inflateInit2: %s",

             stream->msg?stream->msg:"<no message>");

    goto err;
@@ -315,7 +333,8 @@ tor_gzip_uncompress(char **out, size_t *out_len, 
          log_warn(LD_BUG, "Error freeing gzip structures");

          goto err;

        }

        if (inflateInit2(stream, method_bits(method)) != Z_OK) {

        if (inflateInit2(stream,

                         method_bits(method,HIGH_COMPRESSION)) != Z_OK) {

          log_warn(LD_GENERAL, "Error from second inflateInit2: %s",

                   stream->msg?stream->msg:"<no message>");

          goto err;
@@ -426,10 +445,11 @@ struct tor_zlib_state_t { 
 * <b>compress</b>, it's for compression; otherwise it's for

 * decompression. */

tor_zlib_state_t *

tor_zlib_new(int compress, compress_method_t method)

tor_zlib_new(int compress, compress_method_t method,

             zlib_compression_level_t compression_level)

{

  tor_zlib_state_t *out;

  int bits;

  int bits, memlevel;



  if (method == GZIP_METHOD && !is_gzip_supported()) {

    /* Old zlib version don't support gzip in inflateInit2 */
@@ -437,21 +457,29 @@ tor_zlib_new(int compress, compress_method_t method) 
    return NULL;

 }



 if (! compress) {

   /* use this setting for decompression, since we might have the

    * max number of window bits */

   compression_level = HIGH_COMPRESSION;

 }



 out = tor_malloc_zero(sizeof(tor_zlib_state_t));

 out->stream.zalloc = Z_NULL;

 out->stream.zfree = Z_NULL;

 out->stream.opaque = NULL;

 out->compress = compress;

 bits = method_bits(method);

 bits = method_bits(method, compression_level);

 memlevel = get_memlevel(compression_level);

 if (compress) {

   if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,

                    bits, 8, Z_DEFAULT_STRATEGY) != Z_OK)

                    bits, memlevel,

                    Z_DEFAULT_STRATEGY) != Z_OK)

     goto err;

 } else {

   if (inflateInit2(&out->stream, bits) != Z_OK)

     goto err;

 }

 out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8);

 out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel);



 total_zlib_allocation += out->allocation;

src/common/torgzip.h

+11 −1
Original line number Diff line number Diff line
@@ -19,6 +19,15 @@ typedef enum { 
  NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3

} compress_method_t;



/**

 * Enumeration to define tradeoffs between memory usage and compression level.

 * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most

 * memory.

 **/

typedef enum {

  HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION

} zlib_compression_level_t;



int

tor_gzip_compress(char **out, size_t *out_len,

                  const char *in, size_t in_len,
@@ -47,7 +56,8 @@ typedef enum { 
} tor_zlib_output_t;

/** Internal state for an incremental zlib compression/decompression. */

typedef struct tor_zlib_state_t tor_zlib_state_t;

tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method);

tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method,

                               zlib_compression_level_t level);



tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,

                                   char **out, size_t *out_len,

src/or/config.c

+1 −0
Original line number Diff line number Diff line
@@ -2911,6 +2911,7 @@ options_validate(or_options_t *old_options, or_options_t *options, 
  options->MaxMemInQueues =

    compute_real_max_mem_in_queues(options->MaxMemInQueues_raw,

                                   server_mode(options));

  options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3;



  options->AllowInvalid_ = 0;

src/or/directory.c

+28 −5
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include "networkstatus.h"

#include "nodelist.h"

#include "policies.h"

#include "relay.h"

#include "rendclient.h"

#include "rendcommon.h"

#include "rephist.h"
@@ -2539,6 +2540,24 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) 
  return (have >= need_at_least);

}



/** Return the compression level we should use for sending a compressed

 * response of size <b>n_bytes</b>. */

static zlib_compression_level_t

choose_compression_level(ssize_t n_bytes)

{

  if (! have_been_under_memory_pressure()) {

    return HIGH_COMPRESSION; /* we have plenty of RAM. */

  } else if (n_bytes < 0) {

    return HIGH_COMPRESSION; /* unknown; might be big. */

  } else if (n_bytes < 1024) {

    return LOW_COMPRESSION;

  } else if (n_bytes < 2048) {

    return MEDIUM_COMPRESSION;

  } else {

    return HIGH_COMPRESSION;

  }

}



/** Helper function: called when a dirserver gets a complete HTTP GET

 * request.  Look for a request for a directory or for a rendezvous

 * service descriptor.  On finding one, write a response into
@@ -2724,7 +2743,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, 
                               smartlist_len(dir_fps) == 1 ? lifetime : 0);

    conn->fingerprint_stack = dir_fps;

    if (! compressed)

      conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD);

      conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);



    /* Prime the connection with some data. */

    conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
@@ -2812,7 +2831,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, 


    if (smartlist_len(items)) {

      if (compressed) {

        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);

        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,

                                    choose_compression_level(estimated_len));

        SMARTLIST_FOREACH(items, const char *, c,

                 connection_write_to_buf_zlib(c, strlen(c), conn, 0));

        connection_write_to_buf_zlib("", 0, conn, 1);
@@ -2861,7 +2881,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, 
    conn->fingerprint_stack = fps;



    if (compressed)

      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);

      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,

                                      choose_compression_level(dlen));



    connection_dirserv_flushed_some(conn);

    goto done;
@@ -2929,7 +2950,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, 
      }

      write_http_response_header(conn, -1, compressed, cache_lifetime);

      if (compressed)

        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);

        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,

                                        choose_compression_level(dlen));

      /* Prime the connection with some data. */

      connection_dirserv_flushed_some(conn);

    }
@@ -3004,7 +3026,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, 


    write_http_response_header(conn, compressed?-1:len, compressed, 60*60);

    if (compressed) {

      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);

      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,

                                      choose_compression_level(len));

      SMARTLIST_FOREACH(certs, authority_cert_t *, c,

            connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body,

                                         c->cache_info.signed_descriptor_len,