Commit b54a5e70 authored by Nick Mathewson's avatar Nick Mathewson 🐻
Browse files

Split most of dirserv.c into several new modules

In dirauth:
  * bwauth.c reads and uses bandwidth files
  * guardfraction.c reads and uses the guardfraction file
  * reachability.c tests relay reachability
  * recommend_pkg.c handles the recommended-packages lines.
  * recv_descs.c handles fingerprint files and processing incoming
    routerinfos that relays upload to us
  * voteflag.c computes flag thresholds and sets those thresholds on
    routerstatuses when computing votes

In control:
  * fmt_serverstatus.c generates the ancient "v1 server status"
    format that controllers expect.

In nodelist:
  * routerstatus_fmt.c formats routerstatus entries for a consensus,
    a vote, or for the controller.
parent 08e3b88f
......@@ -87,7 +87,9 @@
#else
#include "lib/crypt_ops/crypto_openssl_mgt.h"
#endif
#include "feature/dirauth/bwauth.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/guardfraction.h"
#include "feature/relay/dns.h"
#include "core/or/dos.h"
#include "feature/client/entrynodes.h"
......@@ -141,6 +143,7 @@
#include "lib/evloop/procmon.h"
#include "feature/dirauth/dirvote.h"
#include "feature/dirauth/recommend_pkg.h"
#include "feature/dirauth/mode.h"
#include "core/or/connection_st.h"
......
......@@ -56,6 +56,7 @@ LIBTOR_APP_A_SOURCES = \
src/feature/client/entrynodes.c \
src/feature/client/transports.c \
src/feature/control/control.c \
src/feature/control/fmt_serverstatus.c \
src/feature/dirauth/keypin.c \
src/feature/dircache/conscache.c \
src/feature/dircache/consdiffmgr.c \
......@@ -90,6 +91,7 @@ LIBTOR_APP_A_SOURCES = \
src/feature/nodelist/routerlist.c \
src/feature/nodelist/routerparse.c \
src/feature/nodelist/routerset.c \
src/feature/nodelist/fmt_routerstatus.c \
src/feature/nodelist/torcert.c \
src/feature/relay/dns.c \
src/feature/relay/ext_orport.c \
......@@ -103,6 +105,16 @@ LIBTOR_APP_A_SOURCES = \
src/feature/stats/geoip.c \
src/feature/stats/rephist.c
# These should eventually move into module_dirauth_sources, but for now
# the separation is only in the code location.
LIBTOR_APP_A_SOURCES += \
src/feature/dirauth/bwauth.c \
src/feature/dirauth/guardfraction.c \
src/feature/dirauth/reachability.c \
src/feature/dirauth/recommend_pkg.c \
src/feature/dirauth/process_descs.c \
src/feature/dirauth/voteflags.c
if BUILD_NT_SERVICES
LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
endif
......@@ -222,14 +234,21 @@ noinst_HEADERS += \
src/feature/client/transports.h \
src/feature/control/control.h \
src/feature/control/control_connection_st.h \
src/feature/control/fmt_serverstatus.h \
src/feature/dirauth/bwauth.h \
src/feature/dirauth/dircollate.h \
src/feature/dirauth/dirvote.h \
src/feature/dirauth/guardfraction.h \
src/feature/dirauth/keypin.h \
src/feature/dirauth/mode.h \
src/feature/dirauth/ns_detached_signatures_st.h \
src/feature/dirauth/reachability.h \
src/feature/dirauth/recommend_pkg.h \
src/feature/dirauth/process_descs.h \
src/feature/dirauth/shared_random.h \
src/feature/dirauth/shared_random_state.h \
src/feature/dirauth/vote_microdesc_hash_st.h \
src/feature/dirauth/voteflags.h \
src/feature/dircache/cached_dir_st.h \
src/feature/dircache/conscache.h \
src/feature/dircache/consdiffmgr.h \
......@@ -280,6 +299,7 @@ noinst_HEADERS += \
src/feature/nodelist/routerlist_st.h \
src/feature/nodelist/routerparse.h \
src/feature/nodelist/routerset.h \
src/feature/nodelist/fmt_routerstatus.h \
src/feature/nodelist/routerstatus_st.h \
src/feature/nodelist/signed_descriptor_st.h \
src/feature/nodelist/torcert.h \
......
......@@ -74,6 +74,9 @@
#include "lib/crypt_ops/crypto_rand.h"
#include "feature/dircache/directory.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/bwauth.h"
#include "feature/dirauth/reachability.h"
#include "feature/dirauth/process_descs.h"
#include "feature/relay/dns.h"
#include "feature/client/dnsserv.h"
#include "core/or/dos.h"
......@@ -3663,7 +3666,9 @@ tor_free_all(int postfork)
routerlist_free_all();
networkstatus_free_all();
addressmap_free_all();
dirserv_free_fingerprint_list();
dirserv_free_all();
dirserv_clear_measured_bw_cache();
rend_cache_free_all();
rend_service_authorization_free_all();
rep_hist_free_all();
......
......@@ -41,7 +41,7 @@
#include "feature/control/control.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/reachability.h"
#include "feature/client/entrynodes.h"
#include "feature/stats/geoip.h"
#include "core/mainloop/main.h"
......
......@@ -20,7 +20,6 @@
#include "core/or/or.h"
#include "feature/client/bridges.h"
#include "app/config/config.h"
#include "feature/dircache/dirserv.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
......@@ -39,6 +38,9 @@
#include "feature/nodelist/routerinfo_st.h"
#include "feature/nodelist/routerstatus_st.h"
/** Maximum length of an exit policy summary. */
#define MAX_EXITPOLICY_SUMMARY_LEN 1000
/** Policy that addresses for incoming SOCKS connections must match. */
static smartlist_t *socks_policy = NULL;
/** Policy that addresses for incoming directory connections must match. */
......
......@@ -53,6 +53,7 @@
#include "core/or/connection_edge.h"
#include "core/or/connection_or.h"
#include "feature/control/control.h"
#include "feature/control/fmt_serverstatus.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
#include "feature/dircache/directory.h"
......
/* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2018, The Tor Project, Inc. */
/* See LICENSE for licensing information */
#include "core/or/or.h"
#include "feature/control/fmt_serverstatus.h"
#include "app/config/config.h"
#include "feature/dirauth/voteflags.h"// XXXX remove
#include "feature/nodelist/nodelist.h"
#include "feature/relay/router.h"
#include "feature/nodelist/node_st.h"
#include "feature/nodelist/routerinfo_st.h"
/**
* Allocate and return a description of the status of the server <b>desc</b>,
* for use in a v1-style router-status line. The server is listed
* as running iff <b>is_live</b> is true.
*
* This is deprecated: it's only used for controllers that want outputs in
* the old format.
*/
static char *
list_single_server_status(const routerinfo_t *desc, int is_live)
{
char buf[MAX_NICKNAME_LEN+HEX_DIGEST_LEN+4]; /* !nickname=$hexdigest\0 */
char *cp;
const node_t *node;
tor_assert(desc);
cp = buf;
if (!is_live) {
*cp++ = '!';
}
node = node_get_by_id(desc->cache_info.identity_digest);
if (node && node->is_valid) {
strlcpy(cp, desc->nickname, sizeof(buf)-(cp-buf));
cp += strlen(cp);
*cp++ = '=';
}
*cp++ = '$';
base16_encode(cp, HEX_DIGEST_LEN+1, desc->cache_info.identity_digest,
DIGEST_LEN);
return tor_strdup(buf);
}
/** Based on the routerinfo_ts in <b>routers</b>, allocate the
* contents of a v1-style router-status line, and store it in
* *<b>router_status_out</b>. Return 0 on success, -1 on failure.
*
* If for_controller is true, include the routers with very old descriptors.
*
* This is deprecated: it's only used for controllers that want outputs in
* the old format.
*/
int
list_server_status_v1(smartlist_t *routers, char **router_status_out,
int for_controller)
{
/* List of entries in a router-status style: An optional !, then an optional
* equals-suffixed nickname, then a dollar-prefixed hexdigest. */
smartlist_t *rs_entries;
time_t now = time(NULL);
time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
const or_options_t *options = get_options();
/* We include v2 dir auths here too, because they need to answer
* controllers. Eventually we'll deprecate this whole function;
* see also networkstatus_getinfo_by_purpose(). */
int authdir = authdir_mode_publishes_statuses(options);
tor_assert(router_status_out);
rs_entries = smartlist_new();
SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
tor_assert(node);
if (authdir) {
/* Update router status in routerinfo_t. */
dirserv_set_router_is_running(ri, now);
}
if (for_controller) {
char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
char *cp = name_buf;
if (!node->is_running)
*cp++ = '!';
router_get_verbose_nickname(cp, ri);
smartlist_add_strdup(rs_entries, name_buf);
} else if (ri->cache_info.published_on >= cutoff) {
smartlist_add(rs_entries, list_single_server_status(ri,
node->is_running));
}
} SMARTLIST_FOREACH_END(ri);
*router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL);
SMARTLIST_FOREACH(rs_entries, char *, cp, tor_free(cp));
smartlist_free(rs_entries);
return 0;
}
/* Copyright (c) 2001 Matej Pfajfar.
* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2018, The Tor Project, Inc. */
/* See LICENSE for licensing information */
/**
* \file fmt_serverstatus.h
* \brief Header file for fmt_serverstatus.c.
**/
#ifndef TOR_FMT_SERVERSTATUS_H
#define TOR_FMT_SERVERSTATUS_H
int list_server_status_v1(smartlist_t *routers, char **router_status_out,
int for_controller);
#endif
/* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2018, The Tor Project, Inc. */
/* See LICENSE for licensing information */
/**
* \file bwauth.c
* \brief Code to read and apply bandwidth authority data.
**/
#define BWAUTH_PRIVATE
#include "core/or/or.h"
#include "feature/dirauth/bwauth.h"
#include "app/config/config.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/routerparse.h"
#include "feature/nodelist/routerinfo_st.h"
#include "feature/nodelist/vote_routerstatus_st.h"
#include "lib/encoding/keyval.h"
/** Total number of routers with measured bandwidth; this is set by
* dirserv_count_measured_bs() before the loop in
* dirserv_generate_networkstatus_vote_obj() and checked by
* dirserv_get_credible_bandwidth() and
* dirserv_compute_performance_thresholds() */
static int routers_with_measured_bw = 0;
/** Look through the routerlist, and using the measured bandwidth cache count
* how many measured bandwidths we know. This is used to decide whether we
* ever trust advertised bandwidths for purposes of assigning flags. */
void
dirserv_count_measured_bws(const smartlist_t *routers)
{
/* Initialize this first */
routers_with_measured_bw = 0;
/* Iterate over the routerlist and count measured bandwidths */
SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
/* Check if we know a measured bandwidth for this one */
if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) {
++routers_with_measured_bw;
}
} SMARTLIST_FOREACH_END(ri);
}
/** Return the last-computed result from dirserv_count_mesured_bws(). */
int
dirserv_get_last_n_measured_bws(void)
{
return routers_with_measured_bw;
}
/** Measured bandwidth cache entry */
typedef struct mbw_cache_entry_s {
long mbw_kb;
time_t as_of;
} mbw_cache_entry_t;
/** Measured bandwidth cache - keys are identity_digests, values are
* mbw_cache_entry_t *. */
static digestmap_t *mbw_cache = NULL;
/** Store a measured bandwidth cache entry when reading the measured
* bandwidths file. */
STATIC void
dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
time_t as_of)
{
mbw_cache_entry_t *e = NULL;
tor_assert(parsed_line);
/* Allocate a cache if we need */
if (!mbw_cache) mbw_cache = digestmap_new();
/* Check if we have an existing entry */
e = digestmap_get(mbw_cache, parsed_line->node_id);
/* If we do, we can re-use it */
if (e) {
/* Check that we really are newer, and update */
if (as_of > e->as_of) {
e->mbw_kb = parsed_line->bw_kb;
e->as_of = as_of;
}
} else {
/* We'll have to insert a new entry */
e = tor_malloc(sizeof(*e));
e->mbw_kb = parsed_line->bw_kb;
e->as_of = as_of;
digestmap_set(mbw_cache, parsed_line->node_id, e);
}
}
/** Clear and free the measured bandwidth cache */
void
dirserv_clear_measured_bw_cache(void)
{
if (mbw_cache) {
/* Free the map and all entries */
digestmap_free(mbw_cache, tor_free_);
mbw_cache = NULL;
}
}
/** Scan the measured bandwidth cache and remove expired entries */
STATIC void
dirserv_expire_measured_bw_cache(time_t now)
{
if (mbw_cache) {
/* Iterate through the cache and check each entry */
DIGESTMAP_FOREACH_MODIFY(mbw_cache, k, mbw_cache_entry_t *, e) {
if (now > e->as_of + MAX_MEASUREMENT_AGE) {
tor_free(e);
MAP_DEL_CURRENT(k);
}
} DIGESTMAP_FOREACH_END;
/* Check if we cleared the whole thing and free if so */
if (digestmap_size(mbw_cache) == 0) {
digestmap_free(mbw_cache, tor_free_);
mbw_cache = 0;
}
}
}
/** Query the cache by identity digest, return value indicates whether
* we found it. The bw_out and as_of_out pointers receive the cached
* bandwidth value and the time it was cached if not NULL. */
int
dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out,
time_t *as_of_out)
{
mbw_cache_entry_t *v = NULL;
int rv = 0;
if (mbw_cache && node_id) {
v = digestmap_get(mbw_cache, node_id);
if (v) {
/* Found something */
rv = 1;
if (bw_kb_out) *bw_kb_out = v->mbw_kb;
if (as_of_out) *as_of_out = v->as_of;
}
}
return rv;
}
/** Predicate wrapper for dirserv_query_measured_bw_cache() */
int
dirserv_has_measured_bw(const char *node_id)
{
return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL);
}
/** Get the current size of the measured bandwidth cache */
int
dirserv_get_measured_bw_cache_size(void)
{
if (mbw_cache) return digestmap_size(mbw_cache);
else return 0;
}
/** Return the bandwidth we believe for assigning flags; prefer measured
* over advertised, and if we have above a threshold quantity of measured
* bandwidths, we don't want to ever give flags to unmeasured routers, so
* return 0. */
uint32_t
dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
{
int threshold;
uint32_t bw_kb = 0;
long mbw_kb;
tor_assert(ri);
/* Check if we have a measured bandwidth, and check the threshold if not */
if (!(dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
&mbw_kb, NULL))) {
threshold = get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
if (routers_with_measured_bw > threshold) {
/* Return zero for unmeasured bandwidth if we are above threshold */
bw_kb = 0;
} else {
/* Return an advertised bandwidth otherwise */
bw_kb = router_get_advertised_bandwidth_capped(ri) / 1000;
}
} else {
/* We have the measured bandwidth in mbw */
bw_kb = (uint32_t)mbw_kb;
}
return bw_kb;
}
/**
* Read the measured bandwidth list file, apply it to the list of
* vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
* Returns -1 on error, 0 otherwise.
*/
int
dirserv_read_measured_bandwidths(const char *from_file,
smartlist_t *routerstatuses,
smartlist_t *bw_file_headers)
{
FILE *fp = tor_fopen_cloexec(from_file, "r");
int applied_lines = 0;
time_t file_time, now;
int ok;
/* This flag will be 1 only when the first successful bw measurement line
* has been encountered, so that measured_bw_line_parse don't give warnings
* if there are additional header lines, as introduced in Bandwidth List spec
* version 1.1.0 */
int line_is_after_headers = 0;
int rv = -1;
char *line = NULL;
size_t n = 0;
/* Initialise line, so that we can't possibly run off the end. */
if (fp == NULL) {
log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
from_file);
goto err;
}
/* If fgets fails, line is either unmodified, or indeterminate. */
if (tor_getline(&line,&n,fp) <= 0) {
log_warn(LD_DIRSERV, "Empty bandwidth file");
goto err;
}
if (!strlen(line) || line[strlen(line)-1] != '\n') {
log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
escaped(line));
goto err;
}
line[strlen(line)-1] = '\0';
file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL);
if (!ok) {
log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
escaped(line));
goto err;
}
now = time(NULL);
if ((now - file_time) > MAX_MEASUREMENT_AGE) {
log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
(unsigned)(time(NULL) - file_time));
goto err;
}
/* If timestamp was correct and bw_file_headers is not NULL,
* add timestamp to bw_file_headers */
if (bw_file_headers)
smartlist_add_asprintf(bw_file_headers, "timestamp=%lu",
(unsigned long)file_time);
if (routerstatuses)
smartlist_sort(routerstatuses, compare_vote_routerstatus_entries);
while (!feof(fp)) {
measured_bw_line_t parsed_line;
if (tor_getline(&line, &n, fp) >= 0) {
if (measured_bw_line_parse(&parsed_line, line,
line_is_after_headers) != -1) {
/* This condition will be true when the first complete valid bw line
* has been encountered, which means the end of the header lines. */
line_is_after_headers = 1;
/* Also cache the line for dirserv_get_bandwidth_for_router() */
dirserv_cache_measured_bw(&parsed_line, file_time);
if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0)
applied_lines++;
/* if the terminator is found, it is the end of header lines, set the
* flag but do not store anything */
} else if (strcmp(line, BW_FILE_HEADERS_TERMINATOR) == 0) {
line_is_after_headers = 1;
/* if the line was not a correct relay line nor the terminator and
* the end of the header lines has not been detected yet
* and it is key_value and bw_file_headers did not reach the maximum
* number of headers,
* then assume this line is a header and add it to bw_file_headers */
} else if (bw_file_headers &&
(line_is_after_headers == 0) &&
string_is_key_value(LOG_DEBUG, line) &&
!strchr(line, ' ') &&
(smartlist_len(bw_file_headers)
< MAX_BW_FILE_HEADER_COUNT_IN_VOTE)) {
line[strlen(line)-1] = '\0';
smartlist_add_strdup(bw_file_headers, line);
};
}
}
/* Now would be a nice time to clean the cache, too */
dirserv_expire_measured_bw_cache(now);
log_info(LD_DIRSERV,
"Bandwidth measurement file successfully read. "
"Applied %d measurements.", applied_lines);
rv = 0;
err:
if (line) {
// we need to raw_free this buffer because we got it from tor_getdelim()
raw_free(line);
}
if (fp)
fclose(fp);
return rv;
}
/**
* Helper function to parse out a line in the measured bandwidth file
* into a measured_bw_line_t output structure.
*
* If <b>line_is_after_headers</b> is true, then if we encounter an incomplete
* bw line, return -1 and warn, since we are after the headers and we should
* only parse bw lines. Return 0 otherwise.
*
* If <b>line_is_after_headers</b> is false then it means that we are not past
* the header block yet. If we encounter an incomplete bw line, return -1 but
* don't warn since there could be additional header lines coming. If we
* encounter a proper bw line, return 0 (and we got past the headers).
*/
STATIC int
measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
int line_is_after_headers)
{
char *line = tor_strdup(orig_line);
char *cp = line;
int got_bw = 0;
int got_node_id = 0;
char *strtok_state; /* lame sauce d'jour */
if (strlen(line) == 0) {
log_warn(LD_DIRSERV, "Empty line in bandwidth file");
tor_free(line);
return -1;
}
/* Remove end of line character, so that is not part of the token */
if (line[strlen(line) - 1] == '\n') {
line[strlen(line) - 1] = '\0';
}
cp = tor_strtok_r(cp, " \t", &strtok_state);
if (!cp) {
log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s",
escaped(orig_line));
tor_free(line);
return -1;
}
if (orig_line[strlen(orig_line)-1] != '\n') {
log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
escaped(orig_line));