Commit d38268a8 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Remove v0 hidden service statistics code.

The HSAuthorityRecordStats option was used to track statistics of overall
hidden service usage on the version 0 hidden service authorities. With the
version 2 hidden service directories being deployed and version 0
descriptors being phased out, these statistics are not as useful anymore.

Goodbye, you fine piece of software; my first major code contribution to
Tor.
parent 62c2a5a8
......@@ -83,6 +83,11 @@ Changes in version 0.2.2.6-alpha - 2009-11-19
- Fix a memory leak on directory authorities during voting that was
introduced in 0.2.2.1-alpha. Found via valgrind.
o Removed features:
- Remove the HSAuthorityRecordStats option that version 0 hidden
service authorities could have used to track statistics of overall
hidden service usage.
Changes in version 0.2.2.5-alpha - 2009-10-11
Tor 0.2.2.5-alpha fixes a few compile problems in 0.2.2.4-alpha.
......
......@@ -1197,12 +1197,6 @@ When this option is set in addition to \fBAuthoritativeDirectory\fP, Tor also
accepts and serves hidden service descriptors. (Default: 0)
.LP
.TP
\fBHSAuthorityRecordStats \fR\fB0\fR|\fB1\fR\fP
When this option is set in addition to \fBHSAuthoritativeDir\fP, Tor
periodically (every 15 minutes) writes statistics about hidden service
usage to a file \fBhsusage\fP in its data directory. (Default: 0)
.LP
.TP
\fBHidServDirectoryV2 \fR\fB0\fR|\fB1\fR\fP
When this option is set, Tor accepts and serves v2 hidden service
descriptors. Setting DirPort is not required for this, because clients
......
......@@ -243,7 +243,7 @@ static config_var_t _option_vars[] = {
VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL),
V(HidServAuth, LINELIST, NULL),
V(HSAuthoritativeDir, BOOL, "0"),
V(HSAuthorityRecordStats, BOOL, "0"),
OBSOLETE("HSAuthorityRecordStats"),
V(HttpProxy, STRING, NULL),
V(HttpProxyAuthenticator, STRING, NULL),
V(HttpsProxy, STRING, NULL),
......@@ -3253,10 +3253,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
if (options->AuthoritativeDir && options->ClientOnly)
REJECT("Running as authoritative directory, but ClientOnly also set.");
if (options->HSAuthorityRecordStats && !options->HSAuthoritativeDir)
REJECT("HSAuthorityRecordStats is set but we're not running as "
"a hidden service authority.");
if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly)
REJECT("FetchDirInfoExtraEarly requires that you also set "
"FetchDirInfoEarly");
......
......@@ -2950,18 +2950,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
note_request("/tor/rendezvous?/", desc_len);
/* need to send descp separately, because it may include NULs */
connection_write_to_buf(descp, desc_len, TO_CONN(conn));
/* report successful fetch to statistic */
if (options->HSAuthorityRecordStats) {
hs_usage_note_fetch_total(query, time(NULL));
hs_usage_note_fetch_successful(query, time(NULL));
}
break;
case 0: /* well-formed but not present */
write_http_status_line(conn, 404, "Not found");
/* report (unsuccessful) fetch to statistic */
if (options->HSAuthorityRecordStats) {
hs_usage_note_fetch_total(query, time(NULL));
}
break;
case -1: /* not well-formed */
write_http_status_line(conn, 400, "Bad request");
......
......@@ -824,7 +824,6 @@ run_scheduled_events(time_t now)
static time_t time_to_try_getting_descriptors = 0;
static time_t time_to_reset_descriptor_failures = 0;
static time_t time_to_add_entropy = 0;
static time_t time_to_write_hs_statistics = 0;
static time_t time_to_write_bridge_status_file = 0;
static time_t time_to_downrate_stability = 0;
static time_t time_to_save_stability = 0;
......@@ -1165,12 +1164,6 @@ run_scheduled_events(time_t now)
}
}
/** 10. write hidden service usage statistic to disk */
if (options->HSAuthorityRecordStats && time_to_write_hs_statistics < now) {
hs_usage_write_statistics_to_file(now);
#define WRITE_HSUSAGE_INTERVAL (30*60)
time_to_write_hs_statistics = now+WRITE_HSUSAGE_INTERVAL;
}
/** 10b. write bridge networkstatus file to disk */
if (options->BridgeAuthoritativeDir &&
time_to_write_bridge_status_file < now) {
......@@ -1979,7 +1972,6 @@ tor_free_all(int postfork)
rend_cache_free_all();
rend_service_authorization_free_all();
rep_hist_free_all();
hs_usage_free_all();
dns_free_all();
clear_pending_onions();
circuit_free_all();
......
......@@ -2354,8 +2354,6 @@ typedef struct {
* for version 3 directories? */
int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory
* handle hidden service requests? */
int HSAuthorityRecordStats; /**< Boolean: does this HS authoritative
* directory record statistics? */
int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory
* that's willing to bind names? */
int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative
......
......@@ -1005,7 +1005,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published)
char query[REND_SERVICE_ID_LEN_BASE32+1];
char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */
time_t now;
or_options_t *options = get_options();
tor_assert(rend_cache);
parsed = rend_parse_service_descriptor(desc,desc_len);
if (!parsed) {
......@@ -1040,10 +1039,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published)
rend_service_descriptor_free(parsed);
return -1;
}
/* report novel publication to statistics */
if (published && options->HSAuthorityRecordStats) {
hs_usage_note_publish_total(query, now);
}
tor_snprintf(key, sizeof(key), "0%s", query);
e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key);
if (e && e->parsed->timestamp > parsed->timestamp) {
......@@ -1063,10 +1058,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published)
if (!e) {
e = tor_malloc_zero(sizeof(rend_cache_entry_t));
strmap_set_lc(rend_cache, key, e);
/* report novel publication to statistics */
if (published && options->HSAuthorityRecordStats) {
hs_usage_note_publish_novel(query, now);
}
} else {
rend_service_descriptor_free(e->parsed);
tor_free(e->desc);
......
......@@ -14,7 +14,6 @@
static void bw_arrays_init(void);
static void predicted_ports_init(void);
static void hs_usage_init(void);
/** Total number of bytes currently allocated in fields used by rephist.c. */
uint64_t rephist_total_alloc=0;
......@@ -185,7 +184,6 @@ rep_hist_init(void)
history_map = digestmap_new();
bw_arrays_init();
predicted_ports_init();
hs_usage_init();
}
/** Helper: note that we are no longer connected to the router with history
......@@ -2057,558 +2055,6 @@ rep_hist_free_all(void)
predicted_ports_free();
}
/****************** hidden service usage statistics ******************/
/** How large are the intervals for which we track and report hidden service
* use? */
#define NUM_SECS_HS_USAGE_SUM_INTERVAL (15*60)
/** How far in the past do we remember and publish hidden service use? */
#define NUM_SECS_HS_USAGE_SUM_IS_VALID (24*60*60)
/** How many hidden service usage intervals do we remember? (derived) */
#define NUM_TOTALS_HS_USAGE (NUM_SECS_HS_USAGE_SUM_IS_VALID/ \
NUM_SECS_HS_USAGE_SUM_INTERVAL)
/** List element containing a service id and the count. */
typedef struct hs_usage_list_elem_t {
/** Service id of this elem. */
char service_id[REND_SERVICE_ID_LEN_BASE32+1];
/** Number of occurrences for the given service id. */
uint32_t count;
/* Pointer to next list elem */
struct hs_usage_list_elem_t *next;
} hs_usage_list_elem_t;
/** Ordered list that stores service ids and the number of observations. It is
* ordered by the number of occurrences in descending order. Its purpose is to
* calculate the frequency distribution when the period is over. */
typedef struct hs_usage_list_t {
/* Pointer to the first element in the list. */
hs_usage_list_elem_t *start;
/* Number of total occurrences for all list elements. */
uint32_t total_count;
/* Number of service ids, i.e. number of list elements. */
uint32_t total_service_ids;
} hs_usage_list_t;
/** Tracks service-related observations in the current period and their
* history. */
typedef struct hs_usage_service_related_observation_t {
/** Ordered list that stores service ids and the number of observations in
* the current period. It is ordered by the number of occurrences in
* descending order. Its purpose is to calculate the frequency distribution
* when the period is over. */
hs_usage_list_t *list;
/** Circular arrays that store the history of observations. totals stores all
* observations, twenty (ten, five) the number of observations related to a
* service id being accounted for the top 20 (10, 5) percent of all
* observations. */
uint32_t totals[NUM_TOTALS_HS_USAGE];
uint32_t five[NUM_TOTALS_HS_USAGE];
uint32_t ten[NUM_TOTALS_HS_USAGE];
uint32_t twenty[NUM_TOTALS_HS_USAGE];
} hs_usage_service_related_observation_t;
/** Tracks the history of general period-related observations, i.e. those that
* cannot be related to a specific service id. */
typedef struct hs_usage_general_period_related_observations_t {
/** Circular array that stores the history of observations. */
uint32_t totals[NUM_TOTALS_HS_USAGE];
} hs_usage_general_period_related_observations_t;
/** Keeps information about the current observation period and its relation to
* the histories of observations. */
typedef struct hs_usage_current_observation_period_t {
/** Where do we write the next history entry? */
int next_idx;
/** How many values in history have been set ever? (upper bound!) */
int num_set;
/** When did this period begin? */
time_t start_of_current_period;
/** When does the next period begin? */
time_t start_of_next_period;
} hs_usage_current_observation_period_t;
/** Usage statistics for the current observation period. */
static hs_usage_current_observation_period_t *current_period = NULL;
/** Total number of descriptor publish requests in the current observation
* period. */
static hs_usage_service_related_observation_t *publish_total = NULL;
/** Number of descriptor publish requests for services that have not been
* seen before in the current observation period. */
static hs_usage_service_related_observation_t *publish_novel = NULL;
/** Total number of descriptor fetch requests in the current observation
* period. */
static hs_usage_service_related_observation_t *fetch_total = NULL;
/** Number of successful descriptor fetch requests in the current
* observation period. */
static hs_usage_service_related_observation_t *fetch_successful = NULL;
/** Number of descriptors stored in the current observation period. */
static hs_usage_general_period_related_observations_t *descs = NULL;
/** Creates an empty ordered list element. */
static hs_usage_list_elem_t *
hs_usage_list_elem_new(void)
{
hs_usage_list_elem_t *e;
e = tor_malloc_zero(sizeof(hs_usage_list_elem_t));
rephist_total_alloc += sizeof(hs_usage_list_elem_t);
e->count = 1;
e->next = NULL;
return e;
}
/** Creates an empty ordered list. */
static hs_usage_list_t *
hs_usage_list_new(void)
{
hs_usage_list_t *l;
l = tor_malloc_zero(sizeof(hs_usage_list_t));
rephist_total_alloc += sizeof(hs_usage_list_t);
l->start = NULL;
l->total_count = 0;
l->total_service_ids = 0;
return l;
}
/** Creates an empty structure for storing service-related observations. */
static hs_usage_service_related_observation_t *
hs_usage_service_related_observation_new(void)
{
hs_usage_service_related_observation_t *h;
h = tor_malloc_zero(sizeof(hs_usage_service_related_observation_t));
rephist_total_alloc += sizeof(hs_usage_service_related_observation_t);
h->list = hs_usage_list_new();
return h;
}
/** Creates an empty structure for storing general period-related
* observations. */
static hs_usage_general_period_related_observations_t *
hs_usage_general_period_related_observations_new(void)
{
hs_usage_general_period_related_observations_t *p;
p = tor_malloc_zero(sizeof(hs_usage_general_period_related_observations_t));
rephist_total_alloc+= sizeof(hs_usage_general_period_related_observations_t);
return p;
}
/** Creates an empty structure for storing period-specific information. */
static hs_usage_current_observation_period_t *
hs_usage_current_observation_period_new(void)
{
hs_usage_current_observation_period_t *c;
time_t now;
c = tor_malloc_zero(sizeof(hs_usage_current_observation_period_t));
rephist_total_alloc += sizeof(hs_usage_current_observation_period_t);
now = time(NULL);
c->start_of_current_period = now;
c->start_of_next_period = now + NUM_SECS_HS_USAGE_SUM_INTERVAL;
return c;
}
/** Initializes the structures for collecting hidden service usage data. */
static void
hs_usage_init(void)
{
current_period = hs_usage_current_observation_period_new();
publish_total = hs_usage_service_related_observation_new();
publish_novel = hs_usage_service_related_observation_new();
fetch_total = hs_usage_service_related_observation_new();
fetch_successful = hs_usage_service_related_observation_new();
descs = hs_usage_general_period_related_observations_new();
}
/** Clears the given ordered list by resetting its attributes and releasing
* the memory allocated by its elements. */
static void
hs_usage_list_clear(hs_usage_list_t *lst)
{
/* walk through elements and free memory */
hs_usage_list_elem_t *current = lst->start;
hs_usage_list_elem_t *tmp;
while (current != NULL) {
tmp = current->next;
rephist_total_alloc -= sizeof(hs_usage_list_elem_t);
tor_free(current);
current = tmp;
}
/* reset attributes */
lst->start = NULL;
lst->total_count = 0;
lst->total_service_ids = 0;
return;
}
/** Frees the memory used by the given list. */
static void
hs_usage_list_free(hs_usage_list_t *lst)
{
if (!lst)
return;
hs_usage_list_clear(lst);
rephist_total_alloc -= sizeof(hs_usage_list_t);
tor_free(lst);
}
/** Frees the memory used by the given service-related observations. */
static void
hs_usage_service_related_observation_free(
hs_usage_service_related_observation_t *s)
{
if (!s)
return;
hs_usage_list_free(s->list);
rephist_total_alloc -= sizeof(hs_usage_service_related_observation_t);
tor_free(s);
}
/** Frees the memory used by the given period-specific observations. */
static void
hs_usage_general_period_related_observations_free(
hs_usage_general_period_related_observations_t *s)
{
if (!s)
return;
rephist_total_alloc-=sizeof(hs_usage_general_period_related_observations_t);
tor_free(s);
}
/** Frees the memory used by period-specific information. */
static void
hs_usage_current_observation_period_free(
hs_usage_current_observation_period_t *s)
{
if (!s)
return;
rephist_total_alloc -= sizeof(hs_usage_current_observation_period_t);
tor_free(s);
}
/** Frees all memory that was used for collecting hidden service usage data. */
void
hs_usage_free_all(void)
{
hs_usage_general_period_related_observations_free(descs);
descs = NULL;
hs_usage_service_related_observation_free(fetch_successful);
hs_usage_service_related_observation_free(fetch_total);
hs_usage_service_related_observation_free(publish_novel);
hs_usage_service_related_observation_free(publish_total);
fetch_successful = fetch_total = publish_novel = publish_total = NULL;
hs_usage_current_observation_period_free(current_period);
current_period = NULL;
}
/** Inserts a new occurrence for the given service id to the given ordered
* list. */
static void
hs_usage_insert_value(hs_usage_list_t *lst, const char *service_id)
{
/* search if there is already an elem with same service_id in list */
hs_usage_list_elem_t *current = lst->start;
hs_usage_list_elem_t *previous = NULL;
while (current != NULL && strcasecmp(current->service_id,service_id)) {
previous = current;
current = current->next;
}
/* found an element with same service_id? */
if (current == NULL) {
/* not found! append to end (which could also be the end of a zero-length
* list), don't need to sort (1 is smallest value). */
/* create elem */
hs_usage_list_elem_t *e = hs_usage_list_elem_new();
/* update list attributes (one new elem, one new occurrence) */
lst->total_count++;
lst->total_service_ids++;
/* copy service id to elem */
strlcpy(e->service_id,service_id,sizeof(e->service_id));
/* let either l->start or previously last elem point to new elem */
if (lst->start == NULL) {
/* this is the first elem */
lst->start = e;
} else {
/* there were elems in the list before */
previous->next = e;
}
} else {
/* found! add occurrence to elem and consider resorting */
/* update list attributes (no new elem, but one new occurrence) */
lst->total_count++;
/* add occurrence to elem */
current->count++;
/* is it another than the first list elem? and has previous elem fewer
* count than current? then we need to resort */
if (previous != NULL && previous->count < current->count) {
/* yes! we need to resort */
/* remove current elem first */
previous->next = current->next;
/* can we prepend elem to all other elements? */
if (lst->start->count <= current->count) {
/* yes! prepend elem */
current->next = lst->start;
lst->start = current;
} else {
/* no! walk through list a second time and insert at correct place */
hs_usage_list_elem_t *insert_current = lst->start->next;
hs_usage_list_elem_t *insert_previous = lst->start;
while (insert_current != NULL &&
insert_current->count > current->count) {
insert_previous = insert_current;
insert_current = insert_current->next;
}
/* insert here */
current->next = insert_current;
insert_previous->next = current;
}
}
}
}
/** Writes the current service-related observations to the history array and
* clears the observations of the current period. */
static void
hs_usage_write_service_related_observations_to_history(
hs_usage_current_observation_period_t *p,
hs_usage_service_related_observation_t *h)
{
/* walk through the first 20 % of list elements and calculate frequency
* distributions */
/* maximum indices for the three frequencies */
int five_percent_idx = h->list->total_service_ids/20;
int ten_percent_idx = h->list->total_service_ids/10;
int twenty_percent_idx = h->list->total_service_ids/5;
/* temp values */
uint32_t five_percent = 0;
uint32_t ten_percent = 0;
uint32_t twenty_percent = 0;
/* walk through list */
hs_usage_list_elem_t *current = h->list->start;
int i=0;
while (current != NULL && i <= twenty_percent_idx) {
twenty_percent += current->count;
if (i <= ten_percent_idx)
ten_percent += current->count;
if (i <= five_percent_idx)
five_percent += current->count;
current = current->next;
i++;
}
/* copy frequencies */
h->twenty[p->next_idx] = twenty_percent;
h->ten[p->next_idx] = ten_percent;
h->five[p->next_idx] = five_percent;
/* copy total number of observations */
h->totals[p->next_idx] = h->list->total_count;
/* free memory of old list */
hs_usage_list_clear(h->list);
}
/** Advances to next observation period. */
static void
hs_usage_advance_current_observation_period(void)
{
/* aggregate observations to history, including frequency distribution
* arrays */
hs_usage_write_service_related_observations_to_history(
current_period, publish_total);
hs_usage_write_service_related_observations_to_history(
current_period, publish_novel);
hs_usage_write_service_related_observations_to_history(
current_period, fetch_total);
hs_usage_write_service_related_observations_to_history(
current_period, fetch_successful);
/* write current number of descriptors to descs history */
descs->totals[current_period->next_idx] = rend_cache_size();
/* advance to next period */
current_period->next_idx++;
if (current_period->next_idx == NUM_TOTALS_HS_USAGE)
current_period->next_idx = 0;
if (current_period->num_set < NUM_TOTALS_HS_USAGE)
++current_period->num_set;
current_period->start_of_current_period=current_period->start_of_next_period;
current_period->start_of_next_period += NUM_SECS_HS_USAGE_SUM_INTERVAL;
}
/** Checks if the current period is up to date, and if not, advances it. */
static void
hs_usage_check_if_current_period_is_up_to_date(time_t now)
{
while (now > current_period->start_of_next_period) {
hs_usage_advance_current_observation_period();
}
}
/** Adds a service-related observation, maybe after advancing to next
* observation period. */
static void
hs_usage_add_service_related_observation(
hs_usage_service_related_observation_t *h,
time_t now,
const char *service_id)
{
if (now < current_period->start_of_current_period) {
/* don't record old data */
return;
}
/* check if we are up-to-date */
hs_usage_check_if_current_period_is_up_to_date(now);
/* add observation */
hs_usage_insert_value(h->list, service_id);
}
/** Adds the observation of storing a rendezvous service descriptor to our
* cache in our role as HS authoritative directory. */
void
hs_usage_note_publish_total(const char *service_id, time_t now)
{
hs_usage_add_service_related_observation(publish_total, now, service_id);
}
/** Adds the observation of storing a novel rendezvous service descriptor to
* our cache in our role as HS authoritative directory. */
void
hs_usage_note_publish_novel(const char *service_id, time_t now)
{
hs_usage_add_service_related_observation(publish_novel, now, service_id);
}
/** Adds the observation of being requested for a rendezvous service descriptor
* in our role as HS authoritative directory. */
void
hs_usage_note_fetch_total(const char *service_id, time_t now)
{
hs_usage_add_service_related_observation(fetch_total, now, service_id);
}
/** Adds the observation of being requested for a rendezvous service descriptor
* in our role as HS authoritative directory and being able to answer that
* request successfully. */
void
hs_usage_note_fetch_successful(const char *service_id, time_t now)
{
hs_usage_add_service_related_observation(fetch_successful, now, service_id);
}
/** Writes the given circular array to a string. */
static size_t
hs_usage_format_history(char *buf, size_t len, uint32_t *data)
{
char *cp = buf; /* pointer where we are in the buffer */
int i, n;
if (current_period->num_set <= current_period->next_idx) {
i = 0; /* not been through circular array */
} else {
i = current_period->next_idx;
}
for (n = 0; n < current_period->num_set; ++n,++i) {
if (i >= NUM_TOTALS_HS_USAGE)
i -= NUM_TOTALS_HS_USAGE;
tor_assert(i < NUM_TOTALS_HS_USAGE);
if (n == (current_period->num_set-1))
tor_snprintf(cp, len-(cp-buf), "%d", data[i]);
else
tor_snprintf(cp, len-(cp-buf), "%d,", data[i]);
cp += strlen(cp);
}
return cp-buf;
}
/** Writes the complete usage history as hidden service authoritative directory
* to a string. */
static char *
hs_usage_format_statistics(void)
{
char *buf, *cp, *s = NULL;
char t[ISO_TIME_LEN+1];