Turn into a package

Hopefully everything works first try

.gitignore

 __pycache__
 venv
 passwords.txt
+*.egg-info

requirements.txt

-PySocks==1.6.8
-stem==1.6.0

sbws/__main__.py

+import sbws.commands.client
+import sbws.commands.server
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+
+VERSION = '0.0.1'
+
+
+def create_parser():
+    p = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+    sub = p.add_subparsers(dest='command')
+    sbws.commands.client.gen_parser(sub)
+    sbws.commands.server.gen_parser(sub)
+    return p
+
+
+def main():
+    parser = create_parser()
+    args = parser.parse_args()
+    def_args = [args]
+    def_kwargs = {}
+    known_commands = {
+        'client': {'f': sbws.commands.client.main,
+                   'a': def_args, 'kw': def_kwargs},
+        'server': {'f': sbws.commands.server.main,
+                   'a': def_args, 'kw': def_kwargs},
+    }
+    try:
+        if args.command not in known_commands:
+            parser.print_help()
+        else:
+            comm = known_commands[args.command]
+            exit(comm['f'](*comm['a'], **comm['kw']))
+    except KeyboardInterrupt:
+        print('')

scanner.py → sbws/commands/client.py

-#!/usr/bin/env python3
-from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
-import time
-import socks  # PySocks
-import socket
-import random
+from ..lib.pastlylogger import PastlyLogger
+from ..lib.circuitbuilder import GapsCircuitBuilder as CB
+from ..lib.resultdump import ResultDump
+from ..lib.resultdump import Result
+from ..lib.relaylist import RelayList
+from ..util.simpleauth import is_good_clientside_password_file
+from ..util.simpleauth import authenticate_to_server
+import sbws.util.stem as stem_utils
 from stem.control import EventType
+from argparse import ArgumentDefaultsHelpFormatter
+from multiprocessing.dummy import Pool
 from threading import Event
 from threading import RLock
-from multiprocessing.dummy import Pool
-import util.stem as stem_utils
-from util.simpleauth import is_good_clientside_password_file
-from util.simpleauth import authenticate_to_server
-from lib.circuitbuilder import GapsCircuitBuilder as CB
-from lib.resultdump import ResultDump
-from lib.resultdump import Result
-from lib.relaylist import RelayList
-from lib.pastlylogger import PastlyLogger
+import random
+import socks
+import socket
+import time
+
 
+log = None
 end_event = Event()
 stream_building_lock = RLock()
-log = PastlyLogger(debug='/dev/stdout', overwrite=['debug'], log_threads=True)
 
-# maximum we want to read per read() call
 MAX_RECV_PER_READ = 1*1024*1024
 DOWNLOAD_TIMES = {'toofast': 1, 'min': 5, 'target': 6, 'max': 10}
 DESIRED_RESULTS = 5
@@ -227,7 +226,7 @@ def test_speedtest(args):
     pending_results = []
     relays = rl.relays
     random.shuffle(relays)
-    for target in relays:
+    for target in relays[0:2]:
         callback = result_putter(rd)
         callback_err = result_putter_error(target)
         async_result = pool.apply_async(
@@ -243,37 +242,38 @@ def test_speedtest(args):
     log.notice('Got all results')
 
 
+def gen_parser(sub):
+    p = sub.add_parser('client',
+                       formatter_class=ArgumentDefaultsHelpFormatter)
+    p.add_argument('--control', nargs=2, metavar=('TYPE', 'LOCATION'),
+                   default=['port', '9051'],
+                   help='How to control Tor. Examples: "port 9051" or '
+                   '"socket /var/lib/tor/control"')
+    p.add_argument('--socks-host', default='127.0.0.1', type=str,
+                   help='Host for a local Tor SocksPort')
+    p.add_argument('--socks-port', default=9050, type=int,
+                   help='Port for a local Tor SocksPort')
+    p.add_argument('--server-host', default='127.0.0.1', type=str,
+                   help='Host for a measurement server')
+    p.add_argument('--server-port', default=4444, type=int,
+                   help='Port for a measurement server')
+    p.add_argument('--result-directory', default='dd', type=str,
+                   help='Where to store raw result output')
+    p.add_argument('--threads', default=1, type=int,
+                   help='Number of measurements to make in parallel')
+    p.add_argument('--helper-relay', type=str, required=True,
+                   help='Relay to which to build circuits and is running '
+                   'the server.py')
+    p.add_argument('--password-file', type=str, default='passwords.txt',
+                   help='Read the first line and use it as the password '
+                   'when authenticating to the server.')
+
+
 def main(args):
-    test_speedtest(args)
-
-
-if __name__ == '__main__':
-    parser = ArgumentParser(
-            formatter_class=ArgumentDefaultsHelpFormatter)
-    parser.add_argument('--control', nargs=2, metavar=('TYPE', 'LOCATION'),
-                        default=['port', '9051'],
-                        help='How to control Tor. Examples: "port 9051" or '
-                        '"socket /var/lib/tor/control"')
-    parser.add_argument('--socks-host', default='127.0.0.1', type=str,
-                        help='Host for a local Tor SocksPort')
-    parser.add_argument('--socks-port', default=9050, type=int,
-                        help='Port for a local Tor SocksPort')
-    parser.add_argument('--server-host', default='127.0.0.1', type=str,
-                        help='Host for a measurement server')
-    parser.add_argument('--server-port', default=4444, type=int,
-                        help='Port for a measurement server')
-    parser.add_argument('--result-directory', default='dd', type=str,
-                        help='Where to store raw result output')
-    parser.add_argument('--threads', default=1, type=int,
-                        help='Number of measurements to make in parallel')
-    parser.add_argument('--helper-relay', type=str, required=True,
-                        help='Relay to which to build circuits and is running '
-                        'the server.py')
-    parser.add_argument('--password-file', type=str, default='passwords.txt',
-                        help='Read the first line and use it as the password '
-                        'when authenticating to the server.')
-
-    args = parser.parse_args()
+    global log
+    log = PastlyLogger(debug='/dev/stdout', overwrite=['debug'],
+                       log_threads=True)
+
     if args.threads < 1:
         fail_hard('--threads must be larger than 1')
 
@@ -289,10 +289,8 @@ if __name__ == '__main__':
         fail_hard(error_reason)
 
     try:
-        main(args)
+        test_speedtest(args)
     except KeyboardInterrupt as e:
         raise e
     finally:
         end_event.set()
-
-# pylama:ignore=E265

server.py → sbws/commands/server.py

-#!/usr/bin/env python3
-from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from ..lib.pastlylogger import PastlyLogger
+from ..util.simpleauth import authenticate_client
+from ..util.simpleauth import is_good_serverside_password_file
+from argparse import ArgumentDefaultsHelpFormatter
+from threading import Thread
 import socket
 import time
-from threading import Thread
-from lib.pastlylogger import PastlyLogger
-from util.simpleauth import authenticate_client
-from util.simpleauth import is_good_serverside_password_file
 
-log = PastlyLogger(debug='/dev/stdout', overwrite=['debug'], log_threads=True)
+
+log = None
 
 MAX_SEND_PER_WRITE = 100*1024*1024
 MAX_SEND_PER_WRITE = 4096
 
 
+def gen_parser(sub):
+    p = sub.add_parser('server',
+                       formatter_class=ArgumentDefaultsHelpFormatter)
+    p.add_argument('bind_ip', type=str, default='127.0.0.1')
+    p.add_argument('bind_port', type=int, default=4444)
+    p.add_argument('--password-file', type=str, default='passwords.txt',
+                   help='All lines in this file will be considered '
+                   'valid passwords scanners may use to authenticate.')
+
+
 def fail_hard(*s):
     ''' Optionally log something to stdout ... and then exit as fast as
     possible '''
@@ -95,6 +105,13 @@ def new_thread(args, sock):
 
 
 def main(args):
+    global log
+    log = PastlyLogger(debug='/dev/stdout', overwrite=['debug'],
+                       log_threads=True)
+    valid, error_reason = is_good_serverside_password_file(args.password_file)
+    if not valid:
+        fail_hard(error_reason)
+
     server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     h = (args.bind_ip, args.bind_port)
     log.notice('binding to', h)
@@ -118,20 +135,3 @@ def main(args):
         pass
     finally:
         close_socket(server)
-
-
-if __name__ == '__main__':
-    parser = ArgumentParser(
-            formatter_class=ArgumentDefaultsHelpFormatter)
-    parser.add_argument('bind_ip', type=str, default='127.0.0.1')
-    parser.add_argument('bind_port', type=int, default=4444)
-    parser.add_argument('--password-file', type=str, default='passwords.txt',
-                        help='All lines in this file will be considered '
-                        'valid passwords scanners may use to authenticate.')
-    args = parser.parse_args()
-
-    valid, error_reason = is_good_serverside_password_file(args.password_file)
-    if not valid:
-        fail_hard(error_reason)
-
-    main(args)

lib/circuitbuilder.py → sbws/lib/circuitbuilder.py

 from stem import (CircuitExtensionFailed, InvalidRequest)
 import random
-import util.stem as stem_utils
-from lib.relaylist import RelayList
+import sbws.util.stem as stem_utils
+from .relaylist import RelayList
 
 
 class PathLengthException(Exception):

lib/relaylist.py → sbws/lib/relaylist.py

-import util.stem as stem_utils
+import sbws.util.stem as stem_utils
 from stem import Flag
 import time
 import random

setup.py

+#!/usr/bin/env python3
+# Always prefer setuptools over distutils
+from setuptools import setup, find_packages
+# To use a consistent encoding
+from codecs import open
+import os
+import re
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+with open(os.path.join(here, 'README.md'), encoding='utf-8') as f:
+        long_description = f.read()
+
+
+def get_package_data():
+    # Example that grabs all *.ini files in the cwd and all files in foo/bar
+    # other_files = ['*.ini']
+    # for r, _, fs in os.walk(os.path.join(here, 'foo', 'bar')):
+    #     for f in fs:
+    #         other_files.append(os.path.join(r, f))
+    # return other_files
+    return []
+
+
+def find_version(fname):
+    with open(fname, 'rt') as fd:
+        contents = fd.read()
+        match = re.search(r"^VERSION = ['\"]([^'\"]*)['\"]", contents, re.M)
+        if match:
+            return match.group(1)
+        raise RuntimeError('Unable to find version string')
+
+
+setup(
+    name='sbws',
+    version=find_version('sbws/__main__.py'),
+    description='Simple Bandwidth Scanner',
+    long_description=long_description,
+    author='Matt Traudt',
+    author_email='pastly@torproject.org',
+    # license='MIT',
+    # https://packaging.python.org/tutorials/distributing-packages/#id48
+    # https://pypi.python.org/pypi?%3Aaction=list_classifiers
+    classifiers=[
+        # How mature is this project? Common values are
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        'Development Status :: 3 - Alpha',
+    ],
+    packages=find_packages(),
+    # package_data={
+    #     'foo': get_package_data(),
+    # },
+    keywords='',
+    python_requires='>=3.5',
+    # test_suite='test',
+    entry_points={
+        'console_scripts': [
+            'sbws = sbws.__main__:main',
+        ]
+    },
+    install_requires=[
+        'stem',
+        'pysocks',
+    ],
+    extras_require={
+        'dev': [],
+        'test': [],
+        'doc': ['sphinx'],
+    },
+)