Commit a7ef07b4 authored by Nick Mathewson's avatar Nick Mathewson 🐛
Browse files

r15693@tombo: nickm | 2007-12-25 19:11:29 -0500

 Here, have some terribly clever new buffer code.  It uses a mbuf-like strategy rather than a ring buffer strategy, so it should require far far less extra memory to hold any given amount of data.  Also, it avoids access patterns like x=malloc(1024);x=realloc(x,1048576);x=realloc(x,1024);append_to_freelist(x) that might have been contributing to memory fragmentation.  I've  tested it out a little on peacetime, and it seems to work so far.  If you want to benchmark it for speed, make sure to remove the #define PARANOIA; #define NOINLINE macros at the head of the module.


svn:r12983
parent 1401bc54
Changes in version 0.2.0.16-alpha - 2008-01-??
o Major performance improvements:
- Switch our old ring buffer implementation for one more like that
used by free Unix kernels. The wasted space in a buffer with
1mb of data will now be more like 8k than 1mb. The new
implementation also avoids realloc();realloc(); patterns that
can contribute to memory fragmentation.
Changes in version 0.2.0.15-alpha - 2007-12-25
o Major bugfixes:
- Fix several remotely triggerable asserts based on DirPort requests
......
This diff is collapsed.
......@@ -1873,7 +1873,7 @@ static int
connection_read_to_buf(connection_t *conn, int *max_to_read)
{
int result, at_most = *max_to_read;
size_t bytes_in_buf, more_to_read;
size_t slack_in_buf, more_to_read;
size_t n_read = 0, n_written = 0;
if (at_most == -1) { /* we need to initialize it */
......@@ -1882,11 +1882,11 @@ connection_read_to_buf(connection_t *conn, int *max_to_read)
at_most = connection_bucket_read_limit(conn, time(NULL));
}
bytes_in_buf = buf_capacity(conn->inbuf) - buf_datalen(conn->inbuf);
slack_in_buf = buf_slack(conn->inbuf);
again:
if ((size_t)at_most > bytes_in_buf && bytes_in_buf >= 1024) {
more_to_read = at_most - bytes_in_buf;
at_most = bytes_in_buf;
if ((size_t)at_most > slack_in_buf && slack_in_buf >= 1024) {
more_to_read = at_most - slack_in_buf;
at_most = slack_in_buf;
} else {
more_to_read = 0;
}
......@@ -1997,8 +1997,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read)
connection_buckets_decrement(conn, time(NULL), n_read, n_written);
if (more_to_read && result == at_most) {
bytes_in_buf = buf_capacity(conn->inbuf) - buf_datalen(conn->inbuf);
tor_assert(bytes_in_buf < 1024);
slack_in_buf = buf_slack(conn->inbuf);
at_most = more_to_read;
goto again;
}
......@@ -2785,11 +2784,11 @@ connection_dump_buffer_mem_stats(int severity)
++n_conns_by_type[tp];
if (c->inbuf) {
used_by_type[tp] += buf_datalen(c->inbuf);
alloc_by_type[tp] += buf_capacity(c->inbuf);
alloc_by_type[tp] += buf_allocation(c->inbuf);
}
if (c->outbuf) {
used_by_type[tp] += buf_datalen(c->outbuf);
alloc_by_type[tp] += buf_capacity(c->outbuf);
alloc_by_type[tp] += buf_allocation(c->outbuf);
}
});
for (i=0; i <= _CONN_TYPE_MAX; ++i) {
......
......@@ -1606,13 +1606,13 @@ dumpstats(int severity)
"Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)",
i,
(int)buf_datalen(conn->inbuf),
(int)buf_capacity(conn->inbuf),
(int)buf_allocation(conn->inbuf),
(int)(now - conn->timestamp_lastread));
log(severity,LD_GENERAL,
"Conn %d: %d bytes waiting on outbuf "
"(len %d, last written %d secs ago)",i,
(int)buf_datalen(conn->outbuf),
(int)buf_capacity(conn->outbuf),
(int)buf_allocation(conn->outbuf),
(int)(now - conn->timestamp_lastwritten));
}
circuit_dump_by_conn(conn, severity); /* dump info about all the circuits
......
......@@ -2439,7 +2439,8 @@ void buf_shrink_freelists(int free_all);
void buf_dump_freelist_sizes(int severity);
size_t buf_datalen(const buf_t *buf);
size_t buf_capacity(const buf_t *buf);
size_t buf_allocation(const buf_t *buf);
size_t buf_slack(const buf_t *buf);
const char *_buf_peek_raw_buffer(const buf_t *buf);
int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof);
......
......@@ -127,7 +127,7 @@ test_buffers(void)
if (!(buf = buf_new()))
test_fail();
test_eq(buf_capacity(buf), 4096);
//test_eq(buf_capacity(buf), 4096);
test_eq(buf_datalen(buf), 0);
/****
......@@ -169,25 +169,25 @@ test_buffers(void)
/* Okay, now make sure growing can work. */
buf = buf_new_with_capacity(16);
test_eq(buf_capacity(buf), 16);
//test_eq(buf_capacity(buf), 16);
write_to_buf(str+1, 255, buf);
test_eq(buf_capacity(buf), 256);
//test_eq(buf_capacity(buf), 256);
fetch_from_buf(str2, 254, buf);
test_memeq(str+1, str2, 254);
test_eq(buf_capacity(buf), 256);
//test_eq(buf_capacity(buf), 256);
assert_buf_ok(buf);
write_to_buf(str, 32, buf);
test_eq(buf_capacity(buf), 256);
//test_eq(buf_capacity(buf), 256);
assert_buf_ok(buf);
write_to_buf(str, 256, buf);
assert_buf_ok(buf);
test_eq(buf_capacity(buf), 512);
//test_eq(buf_capacity(buf), 512);
test_eq(buf_datalen(buf), 33+256);
fetch_from_buf(str2, 33, buf);
test_eq(*str2, str[255]);
test_memeq(str2+1, str, 32);
test_eq(buf_capacity(buf), 512);
//test_eq(buf_capacity(buf), 512);
test_eq(buf_datalen(buf), 256);
fetch_from_buf(str2, 256, buf);
test_memeq(str, str2, 256);
......@@ -198,7 +198,7 @@ test_buffers(void)
for (j=0;j<67;++j) {
write_to_buf(str,255, buf);
}
test_eq(buf_capacity(buf), 33668);
//test_eq(buf_capacity(buf), 33668);
test_eq(buf_datalen(buf), 17085);
for (j=0; j < 40; ++j) {
fetch_from_buf(str2, 255,buf);
......@@ -218,7 +218,7 @@ test_buffers(void)
for (j=0;j<80;++j) {
write_to_buf(str,255, buf);
}
test_eq(buf_capacity(buf),33668);
//test_eq(buf_capacity(buf),33668);
for (j=0; j < 120; ++j) {
fetch_from_buf(str2, 255,buf);
test_memeq(str2, str, 255);
......@@ -275,14 +275,14 @@ test_buffers(void)
printf("%s\n", strerror(errno));
test_eq(i, 10);
test_eq(eof, 0);
test_eq(buf_capacity(buf), 4096);
//test_eq(buf_capacity(buf), 4096);
test_eq(buf_datalen(buf), 10);
test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10);
/* Test reading 0 bytes. */
i = read_to_buf(s, 0, buf, &eof);
test_eq(buf_capacity(buf), 512*1024);
//test_eq(buf_capacity(buf), 512*1024);
test_eq(buf_datalen(buf), 10);
test_eq(eof, 0);
test_eq(i, 0);
......@@ -290,7 +290,7 @@ test_buffers(void)
/* Now test when buffer is filled exactly. */
buf2 = buf_new_with_capacity(6);
i = read_to_buf(s, 6, buf2, &eof);
test_eq(buf_capacity(buf2), 6);
//test_eq(buf_capacity(buf2), 6);
test_eq(buf_datalen(buf2), 6);
test_eq(eof, 0);
test_eq(i, 6);
......@@ -300,7 +300,7 @@ test_buffers(void)
/* Now test when buffer is filled with more data to read. */
buf2 = buf_new_with_capacity(32);
i = read_to_buf(s, 128, buf2, &eof);
test_eq(buf_capacity(buf2), 128);
//test_eq(buf_capacity(buf2), 128);
test_eq(buf_datalen(buf2), 32);
test_eq(eof, 0);
test_eq(i, 32);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment