Commit 90674223 authored by Matt Traudt's avatar Matt Traudt
Browse files

Switching to a more legit package style and use config file

parent be30ad7a
......@@ -43,6 +43,22 @@ in the circuits we ask it to build.
See more documentation in [/docs/source/](/docs/source/)
## Configuration
Sbws has two config files it reads.
It first reads the config file containing the default values for almost all
options. If you installed sbws in a virtual environment located at /tmp/venv, then
you will probably find the `config.default.ini` in a place such as
`/tmp/venv/lib/python3.5/site-packages/sbws/`
**You should never edit this file**. You can also click on
[this link](/sbws/config.default.ini) to see the default config file if you're
reading this on GitHub.
Sbws then reads your custom config file. By default, after running `sbws init`,
it is located in `~/.sbws/config.ini`. A configuration option in this file
overwrites the default file found in the default file.
### Build HTML documentation
pip install -e .[doc]
......
......@@ -3,13 +3,19 @@ import sbws.commands.generate
import sbws.commands.init
import sbws.commands.server
import sbws.commands.stats
from sbws.util.config import get_config
from sbws.globals import make_logger
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import os
VERSION = '0.0.1'
def _default_dot_sbws_dname():
home = os.path.expanduser('~')
return os.path.join(home, '.sbws')
def create_parser():
p = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
p.add_argument(
......@@ -18,9 +24,8 @@ def create_parser():
p.add_argument(
'-q', '--quiet', action='count', default=0,
help='Decrease log level verbosity from the configured value')
p.add_argument(
'--data-period', type=int, default=5,
help='Days into the past to consider data fresh')
p.add_argument('-d', '--directory', default=_default_dot_sbws_dname(),
help='Name of the .sbws directory')
sub = p.add_subparsers(dest='command')
sbws.commands.client.gen_parser(sub)
sbws.commands.generate.gen_parser(sub)
......@@ -33,8 +38,9 @@ def create_parser():
def main():
parser = create_parser()
args = parser.parse_args()
log = make_logger(args)
def_args = [args, log]
conf = get_config(args)
log = make_logger(args, conf)
def_args = [args, conf, log]
def_kwargs = {}
known_commands = {
'client': {'f': sbws.commands.client.main,
......
......@@ -25,14 +25,6 @@ import os
end_event = Event()
stream_building_lock = RLock()
# TODO: Store these in a config file. See github#14
# NOTE: move constants to a different file so it's easy to adjust?
# there these values come from?, avg tor speed?
MAX_RECV_PER_READ = 1*1024*1024
DOWNLOAD_TIMES = {'toofast': 1, 'min': 5, 'target': 6, 'max': 10}
DESIRED_RESULTS = 5
INITIAL_READ_REQUEST = 16*1024
def make_socket(socks_host, socks_port):
''' Make a socket that uses the provided socks5 proxy. Note at this point
......@@ -80,13 +72,13 @@ def tell_server_amount(sock, expected_amount):
return True
def timed_recv_from_server(sock, yet_to_read):
def timed_recv_from_server(sock, conf, yet_to_read):
''' Return the time in seconds it took to read <yet_to_read> bytes from
the server. Return None if error '''
assert yet_to_read > 0
start_time = time.time()
while yet_to_read > 0:
limit = min(MAX_RECV_PER_READ, yet_to_read)
limit = min(conf.getint('client', 'max_recv_per_read'), yet_to_read)
try:
read_this_time = len(sock.recv(limit))
except (socket.timeout, ConnectionResetError, BrokenPipeError) as e:
......@@ -122,7 +114,7 @@ def measure_rtt_to_server(sock):
return rtts
def measure_relay(args, cb, rl, relay):
def measure_relay(args, conf, cb, rl, relay):
''' Runs in a worker thread. Measures the given relay. If all measurements
are successful, returns a Result that should get handed off to the
ResultDump. Otherwise returns None.
......@@ -135,7 +127,7 @@ def measure_relay(args, cb, rl, relay):
we built
3. measure the end-to-end RTT many times
4. measure throughput on the built circuit, repeat the following until we
have reached DESIRED_RESULTS
have reached <num_downloads>
4.1. tell the files server the desired amount of bytes to get
4.2. get the bytes and the time it took
4.3. calculate the expected amount of bytes according to:
......@@ -165,7 +157,8 @@ def measure_relay(args, cb, rl, relay):
# circuit when we connect()
stem_utils.add_event_listener(
cb.controller, listener, EventType.STREAM)
s = make_socket(args.socks_host, args.socks_port)
s = make_socket(conf['client']['tor_socks_host'],
conf.getint('client', 'tor_socks_port'))
# This call blocks until we are connected (or give up). We get attched
# to the right circuit in the background.
connected = socket_connect(s, args.server_host, args.server_port)
......@@ -175,7 +168,8 @@ def measure_relay(args, cb, rl, relay):
log.info('Unable to connect to', args.server_host, args.server_port)
cb.close_circuit(circ_id)
return
if not authenticate_to_server(s, args.password_file, log.info):
pw_file = conf.get('paths', 'passwords')
if not authenticate_to_server(s, pw_file, log.info):
log.info('Unable to authenticate to the server')
res = ResultErrorAuth(
relay, circ_fps, args.server_host)
......@@ -192,8 +186,15 @@ def measure_relay(args, cb, rl, relay):
# SECOND: measure throughput on this circuit. Start with what should be a
# small amount
results = []
expected_amount = INITIAL_READ_REQUEST
while len(results) < DESIRED_RESULTS:
expected_amount = conf.getint('client', 'initial_read_request')
num_downloads = conf.getint('client', 'num_downloads')
download_times = {
'toofast': conf.getint('client', 'download_toofast'),
'min': conf.getint('client', 'download_min'),
'target': conf.getint('client', 'download_target'),
'max': conf.getint('client', 'download_max'),
}
while len(results) < num_downloads:
# Tell the server to send us the current expected_amount.
if not tell_server_amount(s, expected_amount):
close_socket(s)
......@@ -201,21 +202,21 @@ def measure_relay(args, cb, rl, relay):
return
# Then read that many bytes from the server and get the time it took to
# do so
result_time = timed_recv_from_server(s, expected_amount)
result_time = timed_recv_from_server(s, conf, expected_amount)
if result_time is None:
close_socket(s)
cb.close_circuit(circ_id)
return
# Adjust amount of bytes to download in the next download
if result_time < DOWNLOAD_TIMES['toofast']:
if result_time < download_times['toofast']:
# Way too fast, greatly increase the amount we ask for
expected_amount = int(expected_amount * 10)
elif result_time < DOWNLOAD_TIMES['min']:
elif result_time < download_times['min']:
# A little too fast, increase the amount we ask for such that it
# will probably take the target amount of time to download
expected_amount = int(
expected_amount * DOWNLOAD_TIMES['target'] / result_time)
elif result_time < DOWNLOAD_TIMES['max']:
expected_amount * download_times['target'] / result_time)
elif result_time < download_times['max']:
# result_time is between min and max, record the result and don't
# change the expected_amount
results.append(
......@@ -224,7 +225,7 @@ def measure_relay(args, cb, rl, relay):
# result_time is too large, decrease the amount we ask for such
# that it will probably take the target amount of time to download
expected_amount = int(
expected_amount * DOWNLOAD_TIMES['target'] / result_time)
expected_amount * download_times['target'] / result_time)
cb.close_circuit(circ_id)
return ResultSuccess(rtts, results, relay, circ_fps, args.server_host)
......@@ -247,16 +248,17 @@ def result_putter_error(target):
return closure
def test_speedtest(args):
controller = stem_utils.init_controller(
port=args.control[1] if args.control[0] == 'port' else None,
path=args.control[1] if args.control[0] == 'socket' else None,
log_fn=log.debug)
cb = CB(args, log, controller=controller)
rl = RelayList(args, log, controller=controller)
rd = ResultDump(args, log, end_event)
rp = RelayPrioritizer(args, log, rl, rd)
max_pending_results = args.threads
def test_speedtest(args, conf):
controller = None
controller, error_msg = stem_utils.init_controller_with_config(conf)
if not controller:
fail_hard(error_msg, log=log)
assert controller
cb = CB(args, conf, log, controller=controller)
rl = RelayList(args, conf, log, controller=controller)
rd = ResultDump(args, conf, log, end_event)
rp = RelayPrioritizer(args, conf, log, rl, rd)
max_pending_results = conf.getint('client', 'measurement_threads')
pool = Pool(max_pending_results)
pending_results = []
while True:
......@@ -265,7 +267,7 @@ def test_speedtest(args):
callback = result_putter(rd)
callback_err = result_putter_error(target)
async_result = pool.apply_async(
measure_relay, [args, cb, rl, target], {},
measure_relay, [args, conf, cb, rl, target], {},
callback, callback_err)
pending_results.append(async_result)
while len(pending_results) >= max_pending_results:
......@@ -276,52 +278,42 @@ def test_speedtest(args):
def gen_parser(sub):
p = sub.add_parser('client',
formatter_class=ArgumentDefaultsHelpFormatter)
p.add_argument('--control', nargs=2, metavar=('TYPE', 'LOCATION'),
default=['port', '9051'],
help='How to control Tor. Examples: "port 9051" or '
'"socket /var/lib/tor/control"')
p.add_argument('--socks-host', default='127.0.0.1', type=str,
help='Host for a local Tor SocksPort')
p.add_argument('--socks-port', default=9050, type=int,
help='Port for a local Tor SocksPort')
p.add_argument('--server-host', default='127.0.0.1', type=str,
help='Host for a measurement server')
p.add_argument('--server-port', default=4444, type=int,
p.add_argument('--server-port', default=31648, type=int,
help='Port for a measurement server')
p.add_argument('--result-directory', default='dd', type=str,
help='Where to store raw result output')
p.add_argument('--threads', default=1, type=int,
help='Number of measurements to make in parallel')
p.add_argument('--helper-relay', type=str, required=True,
help='Relay to which to build circuits and is running '
'the sbws server')
p.add_argument('--password-file', type=str, default='passwords.txt',
help='Read the first line and use it as the password '
'when authenticating to the server.')
def main(args, log_):
def main(args, conf, log_):
global log
log = log_
if not is_initted(os.getcwd()):
if not is_initted(args.directory):
fail_hard('Sbws isn\'t initialized. Try sbws init', log=log)
if args.threads < 1:
fail_hard('--threads must be larger than 1', log=log)
if conf.getint('client', 'measurement_threads') < 1:
fail_hard('Number of measurement threads must be larger than 1',
log=log)
if args.control[0] not in ['port', 'socket']:
if conf['tor']['control_type'] not in ['port', 'socket']:
fail_hard('Must specify either control port or socket. '
'Not "{}"'.format(args.control[0]), log=log)
if args.control[0] == 'port':
args.control[1] = int(args.control[1])
os.makedirs(args.result_directory, exist_ok=True)
'Not "{}"'.format(conf['tor']['control_type'], log=log))
if conf['tor']['control_type'] == 'port':
try:
conf.getint('tor', 'control_location')
except ValueError as e:
fail_hard('Couldn\'t read control port from config:', e, log=log)
os.makedirs(conf['paths']['datadir'], exist_ok=True)
valid, error_reason = is_good_clientside_password_file(args.password_file)
pw_file = conf.get('paths', 'passwords')
valid, error_reason = is_good_clientside_password_file(pw_file)
if not valid:
fail_hard(error_reason)
try:
test_speedtest(args)
test_speedtest(args, conf)
except KeyboardInterrupt as e:
raise e
finally:
......
......@@ -80,8 +80,6 @@ def scale_lines(args, v3bw_lines):
def gen_parser(sub):
p = sub.add_parser('generate',
formatter_class=ArgumentDefaultsHelpFormatter)
p.add_argument('--result-directory', default='dd', type=str,
help='Where result data from the sbws client is stored')
p.add_argument('--output', default='/dev/stdout', type=str,
help='Where to write v3bw file')
p.add_argument('--scale-constant', default=7500, type=int,
......@@ -92,19 +90,21 @@ def gen_parser(sub):
'no scaling')
def main(args, log_):
def main(args, conf, log_):
global log
log = log_
if not is_initted(os.getcwd()):
if not is_initted(args.directory):
fail_hard('Sbws isn\'t initialized. Try sbws init', log=log)
if not os.path.isdir(args.result_directory):
fail_hard(args.result_directory, 'does not exist')
datadir = conf['paths']['datadir']
if not os.path.isdir(datadir):
fail_hard(datadir, 'does not exist')
if args.scale_constant < 1:
fail_hard('--scale-constant must be positive')
data_fnames = sorted(os.listdir(args.result_directory), reverse=True)
data_fnames = sorted(os.listdir(datadir), reverse=True)
data_fnames = data_fnames[0:14]
data_fnames = [os.path.join(args.result_directory, f) for f in data_fnames]
data_fnames = [os.path.join(datadir, f) for f in data_fnames]
data = {}
for fname in data_fnames:
data = read_result_file(fname, data)
......
from sbws.globals import (G_INIT_FILE_MAP, is_initted, fail_hard)
from sbws.util.config import get_user_example_config
from argparse import ArgumentDefaultsHelpFormatter
import os
import shutil
......@@ -8,21 +9,30 @@ def gen_parser(sub):
p = sub.add_parser('init', formatter_class=ArgumentDefaultsHelpFormatter)
def main(args, log_):
def main(args, conf, log_):
global log
log = log_
if is_initted(os.getcwd()):
if is_initted(args.directory):
fail_hard('Directory already seems to be initted', log=log)
dotdir = os.path.join(os.getcwd(), '.sbws')
os.makedirs(dotdir, exist_ok=True)
if not os.path.isdir(args.directory):
log.info('Creating', args.directory)
os.makedirs(args.directory, exist_ok=False)
config_fname = os.path.join(args.directory, 'config.ini')
c = get_user_example_config()
c['paths']['sbws_home'] = args.directory
log.info('Creating', config_fname)
with open(config_fname, 'wt') as fd:
c.write(fd)
for src, dst, ftype in G_INIT_FILE_MAP:
log.info('Creating', dst, '({})'.format(ftype))
dst = os.path.join(args.directory, dst)
if os.path.exists(dst):
log.warn(dst, 'already exists, not overwriting')
continue
if ftype == 'file':
log.info('Creating', dst, '({})'.format(ftype))
try:
shutil.copy(src, dst)
except PermissionError as e:
......
......@@ -5,21 +5,10 @@ from argparse import ArgumentDefaultsHelpFormatter
from threading import Thread
import socket
import time
import os
MAX_SEND_PER_WRITE = 100*1024*1024
MAX_SEND_PER_WRITE = 4096
def gen_parser(sub):
p = sub.add_parser('server',
formatter_class=ArgumentDefaultsHelpFormatter)
p.add_argument('bind_ip', type=str, default='127.0.0.1')
p.add_argument('bind_port', type=int, default=4444)
p.add_argument('--password-file', type=str, default='passwords.txt',
help='All lines in this file will be considered '
'valid passwords scanners may use to authenticate.')
sub.add_parser('server', formatter_class=ArgumentDefaultsHelpFormatter)
def read_line(s):
......@@ -62,11 +51,12 @@ def get_send_amount(sock):
return send_amount
def write_to_client(sock, amount):
def write_to_client(sock, conf, amount):
''' Returns True if successful; else False '''
log.info('Sending client no.', sock.fileno(), amount, 'bytes')
while amount > 0:
amount_this_time = min(MAX_SEND_PER_WRITE, amount)
amount_this_time = min(conf.getint('server', 'max_send_per_write'),
amount)
amount -= amount_this_time
try:
sock.send(b'a' * amount_this_time)
......@@ -76,9 +66,11 @@ def write_to_client(sock, amount):
return True
def new_thread(args, sock):
def new_thread(args, conf, sock):
pw_file = conf.get('paths', 'passwords')
def closure():
if not authenticate_client(sock, args.password_file, log.info):
if not authenticate_client(sock, pw_file, log.info):
log.info('Client did not provide valid auth')
close_socket(sock)
return
......@@ -89,24 +81,25 @@ def new_thread(args, sock):
log.info('Couldn\'t get an amount to send to', sock.fileno())
close_socket(sock)
return
write_to_client(sock, send_amount)
write_to_client(sock, conf, send_amount)
close_socket(sock)
thread = Thread(target=closure)
return thread
def main(args, log_):
def main(args, conf, log_):
global log
log = log_
if not is_initted(os.getcwd()):
if not is_initted(args.directory):
fail_hard('Sbws isn\'t initialized. Try sbws init', log=log)
valid, error_reason = is_good_serverside_password_file(args.password_file)
pw_file = conf.get('paths', 'passwords')
valid, error_reason = is_good_serverside_password_file(pw_file)
if not valid:
fail_hard(error_reason)
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
h = (args.bind_ip, args.bind_port)
h = (conf['server']['bind_ip'], conf.getint('server', 'bind_port'))
log.notice('binding to', h)
while True:
try:
......@@ -122,7 +115,7 @@ def main(args, log_):
while True:
sock, addr = server.accept()
log.info('accepting connection from', addr, 'as', sock.fileno())
t = new_thread(args, sock)
t = new_thread(args, conf, sock)
t.start()
except KeyboardInterrupt:
pass
......
......@@ -46,23 +46,22 @@ def print_stats(data):
def gen_parser(sub):
p = sub.add_parser('stats',
formatter_class=ArgumentDefaultsHelpFormatter)
p.add_argument('--result-directory', default='dd', type=str,
help='Where result data from the sbws client is stored')
sub.add_parser('stats', formatter_class=ArgumentDefaultsHelpFormatter)
def main(args, log_):
def main(args, conf, log_):
global log
log = log_
if not is_initted(os.getcwd()):
if not is_initted(args.directory):
fail_hard('Sbws isn\'t initialized. Try sbws init', log=log)
if not os.path.isdir(args.result_directory):
fail_hard(args.result_directory, 'does not exist')
data_fnames = sorted(os.listdir(args.result_directory), reverse=True)
datadir = conf['paths']['datadir']
if not os.path.isdir(datadir):
fail_hard(datadir, 'does not exist', log=log)
data_fnames = sorted(os.listdir(datadir), reverse=True)
data_fnames = data_fnames[0:14]
data_fnames = [os.path.join(args.result_directory, f) for f in data_fnames]
data_fnames = [os.path.join(datadir, f) for f in data_fnames]
data = {}
for fname in data_fnames:
data = read_result_file(fname, data)
......
[general]
# Days into the past that measurements are considered valid
data_period = 5
# Valid levels, from noisest to quietest: debug, info, notice, warn, error
log_level = notice
[client]
# Maximum number of bytes to read for each sock.recv() call
max_recv_per_read = 1048576
# Limits on what download times are too fast/slow/etc.
download_toofast = 1
download_min = 5
download_target = 6
download_max = 10
# Number of downloads with acceptable times we must have for a relay before
# moving on
num_downloads = 5
# The number of bytes to initially request from the server
initial_read_request = 16384
# Where to connect to Tor for proxying our measurement connections
tor_socks_host = 127.0.0.1
tor_socks_port = 9050
# How many measurements to make in parallel
measurement_threads = 3
[server]
bind_ip = 127.0.0.1
bind_port = 31648
# Maximum number of bytes to write for each sock.send() call
max_send_per_write = 4096
[tor]
control_type = port
control_location = 9051
#control_type = socket
#control_location = /var/run/tor/control
[paths]
passwords = ${sbws_home}/passwords.txt
datadir = ${sbws_home}/datadir
[general]
# Valid levels, from noisest to quietest: debug, info, notice, warn, error
log_level = notice
......@@ -11,12 +11,15 @@ G_INIT_FILE_MAP = [
def is_initted(d):
dotdir = os.path.join(d, '.sbws')
if not os.path.isdir(dotdir):
if not os.path.isdir(d):
return False
for _, fname, _ in G_INIT_FILE_MAP:
fname = os.path.join(d, fname)
if not os.path.exists(fname):
return False
conf_fname = os.path.join(d, 'config.ini')
if not os.path.exists(conf_fname):
return False
return True
......@@ -31,29 +34,37 @@ def fail_hard(*s, log=None):
exit(1)
def make_logger(args): # noqa
def get_log_level_string(args):
arg_level = 2
if args.verbose:
arg_level += args.verbose
if args.quiet:
arg_level -= args.quiet
level = arg_level
if level <= 0:
return 'error'
elif level == 1:
return 'warn'
elif level == 2:
return 'notice'
elif level == 3:
return 'info'
elif level >= 4:
return 'debug'
fail_hard('This should not have been reached.')
def _log_level_string_to_int(s):
if s == 'debug':
return 4
elif s == 'info':
return 3
elif s == 'notice':
return 2
elif s == 'warn':
return 1
elif s == 'error':
return 0
fail_hard('Unknown log level:', s)
def _log_level_int_to_string(i):
if i >= 4:
return 'debug'
elif i == 3:
return 'info'
elif i == 2:
return 'notice'
elif i == 1:
return 'warn'
else: