Merge branch 'refactor_buffers_api_3'

  o Code simplifications and refactoring:

    - Split the portions of the buffer.c module that handle particular

      protocols into separate modules. Part of ticket 23149.

#ifndef TOR_BUFFERS_H

#define TOR_BUFFERS_H

#include "compat.h"

#include "compat.h"

#include "torint.h"

#include "testsupport.h"

typedef struct buf_t buf_t;

struct tor_compress_state_t;

buf_t *buf_new(void);

buf_t *buf_new_with_capacity(size_t size);

size_t buf_get_default_chunk_size(const buf_t *buf);

uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now);

size_t buf_get_total_allocation(void);

int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof,

int buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,

                         int *reached_eof,

                         int *socket_error);

int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf);

int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen);

int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen);

int buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,

                        size_t *buf_flushlen);

int write_to_buf(const char *string, size_t string_len, buf_t *buf);

int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state,

int buf_add(buf_t *buf, const char *string, size_t string_len);

int buf_add_compress(buf_t *buf, struct tor_compress_state_t *state,

                          const char *data, size_t data_len, int done);

int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);

int fetch_from_buf(char *string, size_t string_len, buf_t *buf);

int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto);

int fetch_from_buf_http(buf_t *buf,

                        char **headers_out, size_t max_headerlen,

                        char **body_out, size_t *body_used, size_t max_bodylen,

                        int force_complete);

socks_request_t *socks_request_new(void);

void socks_request_free(socks_request_t *req);

int fetch_from_buf_socks(buf_t *buf, socks_request_t *req,

                         int log_sockstype, int safe_socks);

int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);

int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len);

int peek_buf_has_control0_command(buf_t *buf);

#define PEEK_BUF_STARTSWITH_MAX 16

int peek_buf_startswith(const buf_t *buf, const char *cmd);

int peek_buf_has_http_command(const buf_t *buf);

int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);

void buf_peek(const buf_t *buf, char *string, size_t string_len);

void buf_drain(buf_t *buf, size_t n);

int buf_get_bytes(buf_t *buf, char *string, size_t string_len);

int buf_get_line(buf_t *buf, char *data_out, size_t *data_len);

int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out);

#define PEEK_BUF_STARTSWITH_MAX 16

int buf_peek_startswith(const buf_t *buf, const char *cmd);

int buf_set_to_copy(buf_t **output,

                    const buf_t *input);

void assert_buf_ok(buf_t *buf);

void buf_assert_ok(buf_t *buf);

int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);

void buf_pullup(buf_t *buf, size_t bytes,

                const char **head_out, size_t *len_out);

#ifdef BUFFERS_PRIVATE

STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);

STATIC void buf_pullup(buf_t *buf, size_t bytes);

#ifdef TOR_UNIT_TESTS

void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz);

buf_t *buf_new_with_data(const char *cp, size_t sz);

#endif

STATIC size_t preferred_chunk_size(size_t target);

ATTR_UNUSED STATIC size_t preferred_chunk_size(size_t target);

#define DEBUG_CHUNK_ALLOC

/** A single chunk on a buffer. */

  chunk_t *head; /**< First chunk in the list, or NULL for none. */

  chunk_t *tail; /**< Last chunk in the list, or NULL for none. */

};

#endif

#ifdef BUFFERS_PRIVATE

STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,

                                        size_t *result_out);

chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped);

/** If a read onto the end of a chunk would be smaller than this number, then

 * just start a new chunk. */

#define MIN_READ_LEN 8

/** Return the number of bytes that can be written onto <b>chunk</b> without

 * running out of space. */

static inline size_t

CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)

{

  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);

}

/** Return the next character in <b>chunk</b> onto which data can be appended.

 * If the chunk is full, this might be off the end of chunk->mem. */

static inline char *

CHUNK_WRITE_PTR(chunk_t *chunk)

{

  return chunk->data + chunk->datalen;

}

#endif

#endif

/* Copyright (c) 2001 Matej Pfajfar.

 * Copyright (c) 2001-2004, Roger Dingledine.

 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.

 * Copyright (c) 2007-2017, The Tor Project, Inc. */

/* See LICENSE for licensing information */

#define BUFFERS_PRIVATE

#include "orconfig.h"

#include <stddef.h>

#include "buffers.h"

#include "buffers_tls.h"

#include "compat.h"

#include "compress.h"

#include "util.h"

#include "torint.h"

#include "torlog.h"

#include "tortls.h"

#ifdef HAVE_UNISTD_H

#include <unistd.h>

#endif

/** As read_to_chunk(), but return (negative) error code on error, blocking,

 * or TLS, and the number of bytes read otherwise. */

static inline int

read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,

                  size_t at_most)

{

  int read_result;

  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);

  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);

  if (read_result < 0)

    return read_result;

  buf->datalen += read_result;

  chunk->datalen += read_result;

  return read_result;

}

/** As read_to_buf, but reads from a TLS connection, and returns a TLS

 * status value rather than the number of bytes read.

 *

 * Using TLS on OR connections complicates matters in two ways.

 *

 * First, a TLS stream has its own read buffer independent of the

 * connection's read buffer.  (TLS needs to read an entire frame from

 * the network before it can decrypt any data.  Thus, trying to read 1

 * byte from TLS can require that several KB be read from the network

 * and decrypted.  The extra data is stored in TLS's decrypt buffer.)

 * Because the data hasn't been read by Tor (it's still inside the TLS),

 * this means that sometimes a connection "has stuff to read" even when

 * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is

 * used in connection.c to detect TLS objects with non-empty internal

 * buffers and read from them again.

 *

 * Second, the TLS stream's events do not correspond directly to network

 * events: sometimes, before a TLS stream can read, the network must be

 * ready to write -- or vice versa.

 */

int

buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)

{

  int r = 0;

  size_t total_read = 0;

  check_no_tls_errors();

  if (BUG(buf->datalen >= INT_MAX))

    return -1;

  if (BUG(buf->datalen >= INT_MAX - at_most))

    return -1;

  while (at_most > total_read) {

    size_t readlen = at_most - total_read;

    chunk_t *chunk;

    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {

      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);

      if (readlen > chunk->memlen)

        readlen = chunk->memlen;

    } else {

      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);

      chunk = buf->tail;

      if (cap < readlen)

        readlen = cap;

    }

    r = read_to_chunk_tls(buf, chunk, tls, readlen);

    if (r < 0)

      return r; /* Error */

    tor_assert(total_read+r < INT_MAX);

    total_read += r;

    if ((size_t)r < readlen) /* eof, block, or no more to read. */

      break;

  }

  return (int)total_read;

}

/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk

 * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write

 * more if there is a forced pending write size.)  On success, deduct the

 * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes

 * written on success, and a TOR_TLS error code on failure or blocking.

 */

static inline int

flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,

                size_t sz, size_t *buf_flushlen)

{

  int r;

  size_t forced;

  char *data;

  forced = tor_tls_get_forced_write_size(tls);

  if (forced > sz)

    sz = forced;

  if (chunk) {

    data = chunk->data;

    tor_assert(sz <= chunk->datalen);

  } else {

    data = NULL;

    tor_assert(sz == 0);

  }

  r = tor_tls_write(tls, data, sz);

  if (r < 0)

    return r;

  if (*buf_flushlen > (size_t)r)

    *buf_flushlen -= r;

  else

    *buf_flushlen = 0;

  buf_drain(buf, r);

  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",

            r,(int)*buf_flushlen,(int)buf->datalen);

  return r;

}

/** As buf_flush_to_socket(), but writes data to a TLS connection.  Can write

 * more than <b>flushlen</b> bytes.

 */

int

buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,

              size_t *buf_flushlen)

{

  int r;

  size_t flushed = 0;

  ssize_t sz;

  tor_assert(buf_flushlen);

  tor_assert(*buf_flushlen <= buf->datalen);

  tor_assert(flushlen <= *buf_flushlen);

  sz = (ssize_t) flushlen;

  /* we want to let tls write even if flushlen is zero, because it might

   * have a partial record pending */

  check_no_tls_errors();

  do {

    size_t flushlen0;

    if (buf->head) {

      if ((ssize_t)buf->head->datalen >= sz)

        flushlen0 = sz;

      else

        flushlen0 = buf->head->datalen;

    } else {

      flushlen0 = 0;

    }

    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);

    if (r < 0)

      return r;

    flushed += r;

    sz -= r;

    if (r == 0) /* Can't flush any more now. */

      break;

  } while (sz > 0);

  tor_assert(flushed < INT_MAX);

  return (int)flushed;

}

/* Copyright (c) 2001 Matej Pfajfar.

 * Copyright (c) 2001-2004, Roger Dingledine.

 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.

 * Copyright (c) 2007-2017, The Tor Project, Inc. */

/* See LICENSE for licensing information */

#ifndef TOR_BUFFERS_TLS_H

#define TOR_BUFFERS_TLS_H

struct buf_t;

struct tor_tls_t;

int buf_read_from_tls(struct buf_t *buf,

                      struct tor_tls_t *tls, size_t at_most);

int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls,

                     size_t sz, size_t *buf_flushlen);

#endif