Switching to a more legit package style and use config file

README.md

@@ -43,6 +43,22 @@ in the circuits we ask it to build.
 
 See more documentation in [/docs/source/](/docs/source/)
 
+## Configuration
+
+Sbws has two config files it reads.
+
+It first reads the config file containing the default values for almost all
+options. If you installed sbws in a virtual environment located at /tmp/venv, then
+you will probably find the `config.default.ini` in a place such as
+`/tmp/venv/lib/python3.5/site-packages/sbws/`
+**You should never edit this file**. You can also click on
+[this link](/sbws/config.default.ini) to see the default config file if you're
+reading this on GitHub.
+
+Sbws then reads your custom config file. By default, after running `sbws init`,
+it is located in `~/.sbws/config.ini`. A configuration option in this file
+overwrites the default file found in the default file.
+
 ### Build HTML documentation
 
     pip install -e .[doc]

sbws/__main__.py

@@ -3,13 +3,19 @@ import sbws.commands.generate
 import sbws.commands.init
 import sbws.commands.server
 import sbws.commands.stats
+from sbws.util.config import get_config
 from sbws.globals import make_logger
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-
+import os
 
 VERSION = '0.0.1'
 
 
+def _default_dot_sbws_dname():
+    home = os.path.expanduser('~')
+    return os.path.join(home, '.sbws')
+
+
 def create_parser():
     p = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
     p.add_argument(
@@ -18,9 +24,8 @@ def create_parser():
     p.add_argument(
         '-q', '--quiet', action='count', default=0,
         help='Decrease log level verbosity from the configured value')
-    p.add_argument(
-        '--data-period', type=int, default=5,
-        help='Days into the past to consider data fresh')
+    p.add_argument('-d', '--directory', default=_default_dot_sbws_dname(),
+                   help='Name of the .sbws directory')
     sub = p.add_subparsers(dest='command')
     sbws.commands.client.gen_parser(sub)
     sbws.commands.generate.gen_parser(sub)
@@ -33,8 +38,9 @@ def create_parser():
 def main():
     parser = create_parser()
     args = parser.parse_args()
-    log = make_logger(args)
-    def_args = [args, log]
+    conf = get_config(args)
+    log = make_logger(args, conf)
+    def_args = [args, conf, log]
     def_kwargs = {}
     known_commands = {
         'client': {'f': sbws.commands.client.main,

sbws/commands/client.py

@@ -25,14 +25,6 @@ import os
 end_event = Event()
 stream_building_lock = RLock()
 
-# TODO: Store these in a config file. See github#14
-# NOTE: move constants to a different file so it's easy to adjust?
-# there these values come from?, avg tor speed?
-MAX_RECV_PER_READ = 1*1024*1024
-DOWNLOAD_TIMES = {'toofast': 1, 'min': 5, 'target': 6, 'max': 10}
-DESIRED_RESULTS = 5
-INITIAL_READ_REQUEST = 16*1024
-
 
 def make_socket(socks_host, socks_port):
     ''' Make a socket that uses the provided socks5 proxy. Note at this point
@@ -80,13 +72,13 @@ def tell_server_amount(sock, expected_amount):
     return True
 
 
-def timed_recv_from_server(sock, yet_to_read):
+def timed_recv_from_server(sock, conf, yet_to_read):
     ''' Return the time in seconds it took to read <yet_to_read> bytes from
     the server. Return None if error '''
     assert yet_to_read > 0
     start_time = time.time()
     while yet_to_read > 0:
-        limit = min(MAX_RECV_PER_READ, yet_to_read)
+        limit = min(conf.getint('client', 'max_recv_per_read'), yet_to_read)
         try:
             read_this_time = len(sock.recv(limit))
         except (socket.timeout, ConnectionResetError, BrokenPipeError) as e:
@@ -122,7 +114,7 @@ def measure_rtt_to_server(sock):
     return rtts
 
 
-def measure_relay(args, cb, rl, relay):
+def measure_relay(args, conf, cb, rl, relay):
     ''' Runs in a worker thread. Measures the given relay. If all measurements
     are successful, returns a Result that should get handed off to the
     ResultDump. Otherwise returns None.
@@ -135,7 +127,7 @@ def measure_relay(args, cb, rl, relay):
        we built
     3. measure the end-to-end RTT many times
     4. measure throughput on the built circuit, repeat the following until we
-       have reached DESIRED_RESULTS
+       have reached <num_downloads>
        4.1. tell the files server the desired amount of bytes to get
        4.2. get the bytes and the time it took
        4.3. calculate the expected amount of bytes according to:
@@ -165,7 +157,8 @@ def measure_relay(args, cb, rl, relay):
         # circuit when we connect()
         stem_utils.add_event_listener(
             cb.controller, listener, EventType.STREAM)
-        s = make_socket(args.socks_host, args.socks_port)
+        s = make_socket(conf['client']['tor_socks_host'],
+                        conf.getint('client', 'tor_socks_port'))
         # This call blocks until we are connected (or give up). We get attched
         # to the right circuit in the background.
         connected = socket_connect(s, args.server_host, args.server_port)
@@ -175,7 +168,8 @@ def measure_relay(args, cb, rl, relay):
         log.info('Unable to connect to', args.server_host, args.server_port)
         cb.close_circuit(circ_id)
         return
-    if not authenticate_to_server(s, args.password_file, log.info):
+    pw_file = conf.get('paths', 'passwords')
+    if not authenticate_to_server(s, pw_file, log.info):
         log.info('Unable to authenticate to the server')
         res = ResultErrorAuth(
             relay, circ_fps, args.server_host)
@@ -192,8 +186,15 @@ def measure_relay(args, cb, rl, relay):
     # SECOND: measure throughput on this circuit. Start with what should be a
     # small amount
     results = []
-    expected_amount = INITIAL_READ_REQUEST
-    while len(results) < DESIRED_RESULTS:
+    expected_amount = conf.getint('client', 'initial_read_request')
+    num_downloads = conf.getint('client', 'num_downloads')
+    download_times = {
+        'toofast': conf.getint('client', 'download_toofast'),
+        'min': conf.getint('client', 'download_min'),
+        'target': conf.getint('client', 'download_target'),
+        'max': conf.getint('client', 'download_max'),
+    }
+    while len(results) < num_downloads:
         # Tell the server to send us the current expected_amount.
         if not tell_server_amount(s, expected_amount):
             close_socket(s)
@@ -201,21 +202,21 @@ def measure_relay(args, cb, rl, relay):
             return
         # Then read that many bytes from the server and get the time it took to
         # do so
-        result_time = timed_recv_from_server(s, expected_amount)
+        result_time = timed_recv_from_server(s, conf, expected_amount)
         if result_time is None:
             close_socket(s)
             cb.close_circuit(circ_id)
             return
         # Adjust amount of bytes to download in the next download
-        if result_time < DOWNLOAD_TIMES['toofast']:
+        if result_time < download_times['toofast']:
             # Way too fast, greatly increase the amount we ask for
             expected_amount = int(expected_amount * 10)
-        elif result_time < DOWNLOAD_TIMES['min']:
+        elif result_time < download_times['min']:
             # A little too fast, increase the amount we ask for such that it
             # will probably take the target amount of time to download
             expected_amount = int(
-                expected_amount * DOWNLOAD_TIMES['target'] / result_time)
-        elif result_time < DOWNLOAD_TIMES['max']:
+                expected_amount * download_times['target'] / result_time)
+        elif result_time < download_times['max']:
             # result_time is between min and max, record the result and don't
             # change the expected_amount
             results.append(
@@ -224,7 +225,7 @@ def measure_relay(args, cb, rl, relay):
             # result_time is too large, decrease the amount we ask for such
             # that it will probably take the target amount of time to download
             expected_amount = int(
-                expected_amount * DOWNLOAD_TIMES['target'] / result_time)
+                expected_amount * download_times['target'] / result_time)
     cb.close_circuit(circ_id)
     return ResultSuccess(rtts, results, relay, circ_fps, args.server_host)
 
@@ -247,16 +248,17 @@ def result_putter_error(target):
     return closure
 
 
-def test_speedtest(args):
-    controller = stem_utils.init_controller(
-        port=args.control[1] if args.control[0] == 'port' else None,
-        path=args.control[1] if args.control[0] == 'socket' else None,
-        log_fn=log.debug)
-    cb = CB(args, log, controller=controller)
-    rl = RelayList(args, log, controller=controller)
-    rd = ResultDump(args, log, end_event)
-    rp = RelayPrioritizer(args, log, rl, rd)
-    max_pending_results = args.threads
+def test_speedtest(args, conf):
+    controller = None
+    controller, error_msg = stem_utils.init_controller_with_config(conf)
+    if not controller:
+        fail_hard(error_msg, log=log)
+    assert controller
+    cb = CB(args, conf, log, controller=controller)
+    rl = RelayList(args, conf, log, controller=controller)
+    rd = ResultDump(args, conf, log, end_event)
+    rp = RelayPrioritizer(args, conf, log, rl, rd)
+    max_pending_results = conf.getint('client', 'measurement_threads')
     pool = Pool(max_pending_results)
     pending_results = []
     while True:
@@ -265,7 +267,7 @@ def test_speedtest(args):
             callback = result_putter(rd)
             callback_err = result_putter_error(target)
             async_result = pool.apply_async(
-                measure_relay, [args, cb, rl, target], {},
+                measure_relay, [args, conf, cb, rl, target], {},
                 callback, callback_err)
             pending_results.append(async_result)
             while len(pending_results) >= max_pending_results:
@@ -276,52 +278,42 @@ def test_speedtest(args):
 def gen_parser(sub):
     p = sub.add_parser('client',
                        formatter_class=ArgumentDefaultsHelpFormatter)
-    p.add_argument('--control', nargs=2, metavar=('TYPE', 'LOCATION'),
-                   default=['port', '9051'],
-                   help='How to control Tor. Examples: "port 9051" or '
-                   '"socket /var/lib/tor/control"')
-    p.add_argument('--socks-host', default='127.0.0.1', type=str,
-                   help='Host for a local Tor SocksPort')
-    p.add_argument('--socks-port', default=9050, type=int,
-                   help='Port for a local Tor SocksPort')
     p.add_argument('--server-host', default='127.0.0.1', type=str,
                    help='Host for a measurement server')
-    p.add_argument('--server-port', default=4444, type=int,
+    p.add_argument('--server-port', default=31648, type=int,
                    help='Port for a measurement server')
-    p.add_argument('--result-directory', default='dd', type=str,
-                   help='Where to store raw result output')
-    p.add_argument('--threads', default=1, type=int,
-                   help='Number of measurements to make in parallel')
     p.add_argument('--helper-relay', type=str, required=True,
                    help='Relay to which to build circuits and is running '
                    'the sbws server')
-    p.add_argument('--password-file', type=str, default='passwords.txt',
-                   help='Read the first line and use it as the password '
-                   'when authenticating to the server.')
 
 
-def main(args, log_):
+def main(args, conf, log_):
     global log
     log = log_
-    if not is_initted(os.getcwd()):
+    if not is_initted(args.directory):
         fail_hard('Sbws isn\'t initialized. Try sbws init', log=log)
 
-    if args.threads < 1:
-        fail_hard('--threads must be larger than 1', log=log)
+    if conf.getint('client', 'measurement_threads') < 1:
+        fail_hard('Number of measurement threads must be larger than 1',
+                  log=log)
 
-    if args.control[0] not in ['port', 'socket']:
+    if conf['tor']['control_type'] not in ['port', 'socket']:
         fail_hard('Must specify either control port or socket. '
-                  'Not "{}"'.format(args.control[0]), log=log)
-    if args.control[0] == 'port':
-        args.control[1] = int(args.control[1])
-    os.makedirs(args.result_directory, exist_ok=True)
+                  'Not "{}"'.format(conf['tor']['control_type'], log=log))
+    if conf['tor']['control_type'] == 'port':
+        try:
+            conf.getint('tor', 'control_location')
+        except ValueError as e:
+            fail_hard('Couldn\'t read control port from config:', e, log=log)
+    os.makedirs(conf['paths']['datadir'], exist_ok=True)
 
-    valid, error_reason = is_good_clientside_password_file(args.password_file)
+    pw_file = conf.get('paths', 'passwords')
+    valid, error_reason = is_good_clientside_password_file(pw_file)
     if not valid:
         fail_hard(error_reason)
 
     try:
-        test_speedtest(args)
+        test_speedtest(args, conf)
     except KeyboardInterrupt as e:
         raise e
     finally:

sbws/commands/generate.py

@@ -80,8 +80,6 @@ def scale_lines(args, v3bw_lines):
 def gen_parser(sub):
     p = sub.add_parser('generate',
                        formatter_class=ArgumentDefaultsHelpFormatter)
-    p.add_argument('--result-directory', default='dd', type=str,
-                   help='Where result data from the sbws client is stored')
     p.add_argument('--output', default='/dev/stdout', type=str,
                    help='Where to write v3bw file')
     p.add_argument('--scale-constant', default=7500, type=int,
@@ -92,19 +90,21 @@ def gen_parser(sub):
                    'no scaling')
 
 
-def main(args, log_):
+def main(args, conf, log_):
     global log
     log = log_
-    if not is_initted(os.getcwd()):
+    if not is_initted(args.directory):
         fail_hard('Sbws isn\'t initialized.  Try sbws init', log=log)
-    if not os.path.isdir(args.result_directory):
-        fail_hard(args.result_directory, 'does not exist')
+
+    datadir = conf['paths']['datadir']
+    if not os.path.isdir(datadir):
+        fail_hard(datadir, 'does not exist')
     if args.scale_constant < 1:
         fail_hard('--scale-constant must be positive')
 
-    data_fnames = sorted(os.listdir(args.result_directory), reverse=True)
+    data_fnames = sorted(os.listdir(datadir), reverse=True)
     data_fnames = data_fnames[0:14]
-    data_fnames = [os.path.join(args.result_directory, f) for f in data_fnames]
+    data_fnames = [os.path.join(datadir, f) for f in data_fnames]
     data = {}
     for fname in data_fnames:
         data = read_result_file(fname, data)

sbws/commands/init.py

 from sbws.globals import (G_INIT_FILE_MAP, is_initted, fail_hard)
+from sbws.util.config import get_user_example_config
 from argparse import ArgumentDefaultsHelpFormatter
 import os
 import shutil
@@ -8,21 +9,30 @@ def gen_parser(sub):
     p = sub.add_parser('init', formatter_class=ArgumentDefaultsHelpFormatter)
 
 
-def main(args, log_):
+def main(args, conf, log_):
     global log
     log = log_
-    if is_initted(os.getcwd()):
+    if is_initted(args.directory):
         fail_hard('Directory already seems to be initted', log=log)
 
-    dotdir = os.path.join(os.getcwd(), '.sbws')
-    os.makedirs(dotdir, exist_ok=True)
+    if not os.path.isdir(args.directory):
+        log.info('Creating', args.directory)
+        os.makedirs(args.directory, exist_ok=False)
+
+    config_fname = os.path.join(args.directory, 'config.ini')
+    c = get_user_example_config()
+    c['paths']['sbws_home'] = args.directory
+    log.info('Creating', config_fname)
+    with open(config_fname, 'wt') as fd:
+        c.write(fd)
 
     for src, dst, ftype in G_INIT_FILE_MAP:
-        log.info('Creating', dst, '({})'.format(ftype))
+        dst = os.path.join(args.directory, dst)
         if os.path.exists(dst):
             log.warn(dst, 'already exists, not overwriting')
             continue
         if ftype == 'file':
+            log.info('Creating', dst, '({})'.format(ftype))
             try:
                 shutil.copy(src, dst)
             except PermissionError as e:

sbws/commands/server.py

@@ -5,21 +5,10 @@ from argparse import ArgumentDefaultsHelpFormatter
 from threading import Thread
 import socket
 import time
-import os
-
-
-MAX_SEND_PER_WRITE = 100*1024*1024
-MAX_SEND_PER_WRITE = 4096
 
 
 def gen_parser(sub):
-    p = sub.add_parser('server',
-                       formatter_class=ArgumentDefaultsHelpFormatter)
-    p.add_argument('bind_ip', type=str, default='127.0.0.1')
-    p.add_argument('bind_port', type=int, default=4444)
-    p.add_argument('--password-file', type=str, default='passwords.txt',
-                   help='All lines in this file will be considered '
-                   'valid passwords scanners may use to authenticate.')
+    sub.add_parser('server', formatter_class=ArgumentDefaultsHelpFormatter)
 
 
 def read_line(s):
@@ -62,11 +51,12 @@ def get_send_amount(sock):
     return send_amount
 
 
-def write_to_client(sock, amount):
+def write_to_client(sock, conf, amount):
     ''' Returns True if successful; else False '''
     log.info('Sending client no.', sock.fileno(), amount, 'bytes')
     while amount > 0:
-        amount_this_time = min(MAX_SEND_PER_WRITE, amount)
+        amount_this_time = min(conf.getint('server', 'max_send_per_write'),
+                               amount)
         amount -= amount_this_time
         try:
             sock.send(b'a' * amount_this_time)
@@ -76,9 +66,11 @@ def write_to_client(sock, amount):
     return True
 
 
-def new_thread(args, sock):
+def new_thread(args, conf, sock):
+    pw_file = conf.get('paths', 'passwords')
+
     def closure():
-        if not authenticate_client(sock, args.password_file, log.info):
+        if not authenticate_client(sock, pw_file, log.info):
             log.info('Client did not provide valid auth')
             close_socket(sock)
             return
@@ -89,24 +81,25 @@ def new_thread(args, sock):
                 log.info('Couldn\'t get an amount to send to', sock.fileno())
                 close_socket(sock)
                 return
-            write_to_client(sock, send_amount)
+            write_to_client(sock, conf, send_amount)
         close_socket(sock)
     thread = Thread(target=closure)
     return thread
 
 
-def main(args, log_):
+def main(args, conf, log_):
     global log
     log = log_
-    if not is_initted(os.getcwd()):
+    if not is_initted(args.directory):
         fail_hard('Sbws isn\'t initialized. Try sbws init', log=log)
 
-    valid, error_reason = is_good_serverside_password_file(args.password_file)
+    pw_file = conf.get('paths', 'passwords')
+    valid, error_reason = is_good_serverside_password_file(pw_file)
     if not valid:
         fail_hard(error_reason)
 
     server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    h = (args.bind_ip, args.bind_port)
+    h = (conf['server']['bind_ip'], conf.getint('server', 'bind_port'))
     log.notice('binding to', h)
     while True:
         try:
@@ -122,7 +115,7 @@ def main(args, log_):
         while True:
             sock, addr = server.accept()
             log.info('accepting connection from', addr, 'as', sock.fileno())
-            t = new_thread(args, sock)
+            t = new_thread(args, conf, sock)
             t.start()
     except KeyboardInterrupt:
         pass

sbws/commands/stats.py

@@ -46,23 +46,22 @@ def print_stats(data):
 
 
 def gen_parser(sub):
-    p = sub.add_parser('stats',
-                       formatter_class=ArgumentDefaultsHelpFormatter)
-    p.add_argument('--result-directory', default='dd', type=str,
-                   help='Where result data from the sbws client is stored')
+    sub.add_parser('stats', formatter_class=ArgumentDefaultsHelpFormatter)
 
 
-def main(args, log_):
+def main(args, conf, log_):
     global log
     log = log_
-    if not is_initted(os.getcwd()):
+    if not is_initted(args.directory):
         fail_hard('Sbws isn\'t initialized. Try sbws init', log=log)
-    if not os.path.isdir(args.result_directory):
-        fail_hard(args.result_directory, 'does not exist')
 
-    data_fnames = sorted(os.listdir(args.result_directory), reverse=True)
+    datadir = conf['paths']['datadir']
+    if not os.path.isdir(datadir):
+        fail_hard(datadir, 'does not exist', log=log)
+
+    data_fnames = sorted(os.listdir(datadir), reverse=True)
     data_fnames = data_fnames[0:14]
-    data_fnames = [os.path.join(args.result_directory, f) for f in data_fnames]
+    data_fnames = [os.path.join(datadir, f) for f in data_fnames]
     data = {}
     for fname in data_fnames:
         data = read_result_file(fname, data)

sbws/config.default.ini

+[general]
+# Days into the past that measurements are considered valid
+data_period = 5
+# Valid levels, from noisest to quietest: debug, info, notice, warn, error
+log_level = notice
+
+[client]
+# Maximum number of bytes to read for each sock.recv() call
+max_recv_per_read = 1048576
+# Limits on what download times are too fast/slow/etc.
+download_toofast = 1
+download_min = 5
+download_target = 6
+download_max = 10
+# Number of downloads with acceptable times we must have for a relay before
+# moving on
+num_downloads = 5
+# The number of bytes to initially request from the server
+initial_read_request = 16384
+# Where to connect to Tor for proxying our measurement connections
+tor_socks_host = 127.0.0.1
+tor_socks_port = 9050
+# How many measurements to make in parallel
+measurement_threads = 3
+
+[server]
+bind_ip = 127.0.0.1
+bind_port = 31648
+# Maximum number of bytes to write for each sock.send() call
+max_send_per_write = 4096
+
+[tor]
+control_type = port
+control_location = 9051
+#control_type = socket
+#control_location = /var/run/tor/control

sbws/config.example.ini

+[paths]
+passwords = ${sbws_home}/passwords.txt
+datadir = ${sbws_home}/datadir
+
+[general]
+# Valid levels, from noisest to quietest: debug, info, notice, warn, error
+log_level = notice

sbws/globals.py

@@ -11,12 +11,15 @@ G_INIT_FILE_MAP = [
 
 
 def is_initted(d):
-    dotdir = os.path.join(d, '.sbws')
-    if not os.path.isdir(dotdir):
+    if not os.path.isdir(d):
         return False
     for _, fname, _ in G_INIT_FILE_MAP:
+        fname = os.path.join(d, fname)
         if not os.path.exists(fname):
             return False
+    conf_fname = os.path.join(d, 'config.ini')
+    if not os.path.exists(conf_fname):
+        return False
     return True
 
 
@@ -31,29 +34,37 @@ def fail_hard(*s, log=None):
     exit(1)
 
 
-def make_logger(args): # noqa
-    def get_log_level_string(args):
-        arg_level = 2
-        if args.verbose:
-            arg_level += args.verbose
-        if args.quiet:
-            arg_level -= args.quiet
-        level = arg_level
-        if level <= 0:
-            return 'error'
-        elif level == 1:
-            return 'warn'
-        elif level == 2:
-            return 'notice'
-        elif level == 3:
-            return 'info'
-        elif level >= 4:
-            return 'debug'
-        fail_hard('This should not have been reached.')
+def _log_level_string_to_int(s):
+    if s == 'debug':
+        return 4
+    elif s == 'info':
+        return 3
+    elif s == 'notice':
+        return 2
+    elif s == 'warn':
+        return 1
+    elif s == 'error':
+        return 0
+    fail_hard('Unknown log level:', s)
+
+
+def _log_level_int_to_string(i):
+    if i >= 4:
+        return 'debug'
+    elif i == 3:
+        return 'info'
+    elif i == 2:
+        return 'notice'
+    elif i == 1:
+        return 'warn'
+    else: