Commit 7248afc0 authored by juga  's avatar juga
Browse files

fix: Reformat all with black

parent 7d26503c
from ._version import get_versions
__version__ = get_versions()['version']
__version__ = get_versions()["version"]
del get_versions
import threading # noqa
......@@ -19,6 +20,7 @@ class Settings:
should be initialized here.
"""
def __init__(self):
# update this dict from globals (but only for ALL_CAPS settings)
for setting in dir(globals):
......@@ -27,9 +29,9 @@ class Settings:
self.end_event = threading.Event()
def init_http_headers(self, nickname, uuid, tor_version):
self.HTTP_HEADERS['Tor-Bandwidth-Scanner-Nickname'] = nickname
self.HTTP_HEADERS['Tor-Bandwidth-Scanner-UUID'] = uuid
self.HTTP_HEADERS['User-Agent'] += tor_version
self.HTTP_HEADERS["Tor-Bandwidth-Scanner-Nickname"] = nickname
self.HTTP_HEADERS["Tor-Bandwidth-Scanner-UUID"] = uuid
self.HTTP_HEADERS["User-Agent"] += tor_version
def set_end_event(self):
self.end_event.set()
......
# This file helps to compute a version number in source trees obtained from
# git-archive tarball (such as those provided by githubs download-from-tag
# feature). Distribution tarballs (built by setup.py sdist) and build
......@@ -58,17 +57,20 @@ HANDLERS = {}
def register_vcs_handler(vcs, method): # decorator
"""Create decorator to mark a method as the handler of a VCS."""
def decorate(f):
"""Store f in HANDLERS[vcs][method]."""
if vcs not in HANDLERS:
HANDLERS[vcs] = {}
HANDLERS[vcs][method] = f
return f
return decorate
def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
env=None):
def run_command(
commands, args, cwd=None, verbose=False, hide_stderr=False, env=None
):
"""Call the given command(s)."""
assert isinstance(commands, list)
p = None
......@@ -76,10 +78,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
try:
dispcmd = str([c] + args)
# remember shell=False, so use git.cmd on windows, not just git
p = subprocess.Popen([c] + args, cwd=cwd, env=env,
stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr
else None))
p = subprocess.Popen(
[c] + args,
cwd=cwd,
env=env,
stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr else None),
)
break
except EnvironmentError:
e = sys.exc_info()[1]
......@@ -114,16 +119,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
for i in range(3):
dirname = os.path.basename(root)
if dirname.startswith(parentdir_prefix):
return {"version": dirname[len(parentdir_prefix):],
"full-revisionid": None,
"dirty": False, "error": None, "date": None}
return {
"version": dirname[len(parentdir_prefix) :],
"full-revisionid": None,
"dirty": False,
"error": None,
"date": None,
}
else:
rootdirs.append(root)
root = os.path.dirname(root) # up a level
if verbose:
print("Tried directories %s but none started with prefix %s" %
(str(rootdirs), parentdir_prefix))
print(
"Tried directories %s but none started with prefix %s"
% (str(rootdirs), parentdir_prefix)
)
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
......@@ -183,7 +194,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
TAG = "tag: "
tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)])
if not tags:
# Either we're using git < 1.8.3, or there really are no tags. We use
# a heuristic: assume all version tags have a digit. The old git %d
......@@ -192,7 +203,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
# between branches and tags. By ignoring refnames without digits, we
# filter out many common branch names like "release" and
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
tags = set([r for r in refs if re.search(r"\d", r)])
if verbose:
print("discarding '%s', no digits" % ",".join(refs - tags))
if verbose:
......@@ -200,19 +211,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
for ref in sorted(tags):
# sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix):
r = ref[len(tag_prefix):]
r = ref[len(tag_prefix) :]
if verbose:
print("picking %s" % r)
return {"version": r,
"full-revisionid": keywords["full"].strip(),
"dirty": False, "error": None,
"date": date}
return {
"version": r,
"full-revisionid": keywords["full"].strip(),
"dirty": False,
"error": None,
"date": date,
}
# no suitable tags, so version is "0+unknown", but full hex is still there
if verbose:
print("no suitable tags, using unknown + full revision id")
return {"version": "0+unknown",
"full-revisionid": keywords["full"].strip(),
"dirty": False, "error": "no suitable tags", "date": None}
return {
"version": "0+unknown",
"full-revisionid": keywords["full"].strip(),
"dirty": False,
"error": "no suitable tags",
"date": None,
}
@register_vcs_handler("git", "pieces_from_vcs")
......@@ -227,8 +245,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
if sys.platform == "win32":
GITS = ["git.cmd", "git.exe"]
out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
hide_stderr=True)
out, rc = run_command(
GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True
)
if rc != 0:
if verbose:
print("Directory %s not under git control" % root)
......@@ -236,10 +255,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
# if there isn't one, this yields HEX[-dirty] (no NUM)
describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
"--always", "--long",
"--match", "%s*" % tag_prefix],
cwd=root)
describe_out, rc = run_command(
GITS,
[
"describe",
"--tags",
"--dirty",
"--always",
"--long",
"--match",
"%s*" % tag_prefix,
],
cwd=root,
)
# --long was added in git-1.5.5
if describe_out is None:
raise NotThisMethod("'git describe' failed")
......@@ -262,17 +290,18 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
dirty = git_describe.endswith("-dirty")
pieces["dirty"] = dirty
if dirty:
git_describe = git_describe[:git_describe.rindex("-dirty")]
git_describe = git_describe[: git_describe.rindex("-dirty")]
# now we have TAG-NUM-gHEX or HEX
if "-" in git_describe:
# TAG-NUM-gHEX
mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe)
if not mo:
# unparseable. Maybe git-describe is misbehaving?
pieces["error"] = ("unable to parse git-describe output: '%s'"
% describe_out)
pieces["error"] = (
"unable to parse git-describe output: '%s'" % describe_out
)
return pieces
# tag
......@@ -281,10 +310,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
if verbose:
fmt = "tag '%s' doesn't start with prefix '%s'"
print(fmt % (full_tag, tag_prefix))
pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
% (full_tag, tag_prefix))
pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (
full_tag,
tag_prefix,
)
return pieces
pieces["closest-tag"] = full_tag[len(tag_prefix):]
pieces["closest-tag"] = full_tag[len(tag_prefix) :]
# distance: number of commits since tag
pieces["distance"] = int(mo.group(2))
......@@ -295,13 +326,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
else:
# HEX: no tags
pieces["closest-tag"] = None
count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
cwd=root)
count_out, rc = run_command(
GITS, ["rev-list", "HEAD", "--count"], cwd=root
)
pieces["distance"] = int(count_out) # total number of commits
# commit date: see ISO-8601 comment in git_versions_from_keywords()
date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
cwd=root)[0].strip()
date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[
0
].strip()
# Use only the last line. Previous lines may contain GPG signature
# information.
date = date.splitlines()[-1]
......@@ -335,8 +368,7 @@ def render_pep440(pieces):
rendered += ".dirty"
else:
# exception #1
rendered = "0+untagged.%d.g%s" % (pieces["distance"],
pieces["short"])
rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"])
if pieces["dirty"]:
rendered += ".dirty"
return rendered
......@@ -450,11 +482,13 @@ def render_git_describe_long(pieces):
def render(pieces, style):
"""Render the given version pieces into the requested style."""
if pieces["error"]:
return {"version": "unknown",
"full-revisionid": pieces.get("long"),
"dirty": None,
"error": pieces["error"],
"date": None}
return {
"version": "unknown",
"full-revisionid": pieces.get("long"),
"dirty": None,
"error": pieces["error"],
"date": None,
}
if not style or style == "default":
style = "pep440" # the default
......@@ -474,9 +508,13 @@ def render(pieces, style):
else:
raise ValueError("unknown style '%s'" % style)
return {"version": rendered, "full-revisionid": pieces["long"],
"dirty": pieces["dirty"], "error": None,
"date": pieces.get("date")}
return {
"version": rendered,
"full-revisionid": pieces["long"],
"dirty": pieces["dirty"],
"error": None,
"date": pieces.get("date"),
}
def get_versions():
......@@ -490,8 +528,9 @@ def get_versions():
verbose = cfg.verbose
try:
return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
verbose)
return git_versions_from_keywords(
get_keywords(), cfg.tag_prefix, verbose
)
except NotThisMethod:
pass
......@@ -500,13 +539,16 @@ def get_versions():
# versionfile_source is the relative path from the top of the source
# tree (where the .git directory might live) to this file. Invert
# this to find the root from __file__.
for i in cfg.versionfile_source.split('/'):
for i in cfg.versionfile_source.split("/"):
root = os.path.dirname(root)
except NameError:
return {"version": "0+unknown", "full-revisionid": None,
"dirty": None,
"error": "unable to find root of source tree",
"date": None}
return {
"version": "0+unknown",
"full-revisionid": None,
"dirty": None,
"error": "unable to find root of source tree",
"date": None,
}
try:
pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
......@@ -520,6 +562,10 @@ def get_versions():
except NotThisMethod:
pass
return {"version": "0+unknown", "full-revisionid": None,
"dirty": None,
"error": "unable to compute version", "date": None}
return {
"version": "0+unknown",
"full-revisionid": None,
"dirty": None,
"error": "unable to compute version",
"date": None,
}
......@@ -17,22 +17,30 @@ log = logging.getLogger(__name__)
def gen_parser(sub):
'''
"""
Helper function for the broader argument parser generating code that adds
in all the possible command line arguments for the cleanup command.
:param argparse._SubParsersAction sub: what to add a sub-parser to
'''
d = 'Compress and delete results and/or v3bw files old files.' \
'Configuration options are read to determine which are old files'
p = sub.add_parser('cleanup', description=d,
formatter_class=ArgumentDefaultsHelpFormatter)
p.add_argument('--dry-run', action='store_true',
help='Don\'t actually compress or delete anything')
p.add_argument('--no-results', action='store_true',
help='Do not clean results files')
p.add_argument('--no-v3bw', action='store_true',
help='Do not clean v3bw files')
"""
d = (
"Compress and delete results and/or v3bw files old files."
"Configuration options are read to determine which are old files"
)
p = sub.add_parser(
"cleanup", description=d, formatter_class=ArgumentDefaultsHelpFormatter
)
p.add_argument(
"--dry-run",
action="store_true",
help="Don't actually compress or delete anything",
)
p.add_argument(
"--no-results", action="store_true", help="Do not clean results files"
)
p.add_argument(
"--no-v3bw", action="store_true", help="Do not clean v3bw files"
)
def _get_files_mtime_older_than(dname, days_delta, extensions):
......@@ -43,7 +51,7 @@ def _get_files_mtime_older_than(dname, days_delta, extensions):
assert isinstance(extensions, list)
for ext in extensions:
assert isinstance(ext, str)
assert ext[0] == '.'
assert ext[0] == "."
# Determine oldest allowed date
today = datetime.utcfromtimestamp(time.time())
oldest_day = today - timedelta(days=days_delta)
......@@ -52,13 +60,17 @@ def _get_files_mtime_older_than(dname, days_delta, extensions):
fname = os.path.join(root, f)
_, ext = os.path.splitext(fname)
if ext not in extensions:
log.debug('Ignoring %s because its extension is not in '
'%s', fname, extensions)
log.debug(
"Ignoring %s because its extension is not in " "%s",
fname,
extensions,
)
continue
# using file modification time instead of parsing the name
# of the file.
filedt = unixts_to_dt_obj(
os.stat(fname, follow_symlinks=False).st_mtime)
os.stat(fname, follow_symlinks=False).st_mtime
)
if filedt < oldest_day:
yield fname
......@@ -69,7 +81,7 @@ def _delete_files(dname, files, dry_run=True):
assert isinstance(files, types.GeneratorType)
with DirectoryLock(dname):
for fname in files:
log.info('Deleting %s', fname)
log.info("Deleting %s", fname)
assert os.path.commonprefix([dname, fname]) == dname
if not dry_run:
os.remove(fname)
......@@ -81,13 +93,13 @@ def _compress_files(dname, files, dry_run=True):
assert isinstance(files, types.GeneratorType)
with DirectoryLock(dname):
for fname in files:
log.info('Compressing %s', fname)
log.info("Compressing %s", fname)
assert os.path.commonprefix([dname, fname]) == dname
if dry_run:
continue
with open(fname, 'rt') as in_fd:
out_fname = fname + '.gz'
with gzip.open(out_fname, 'wt') as out_fd:
with open(fname, "rt") as in_fd:
out_fname = fname + ".gz"
with gzip.open(out_fname, "wt") as out_fd:
shutil.copyfileobj(in_fd, out_fd)
os.remove(fname)
......@@ -95,60 +107,64 @@ def _compress_files(dname, files, dry_run=True):
def _check_validity_periods_v3bw(compress_after_days, delete_after_days):
if 1 <= compress_after_days and compress_after_days < delete_after_days:
return True
fail_hard("v3bw files should only be compressed after 1 day and deleted "
"after a bigger number of days.")
fail_hard(
"v3bw files should only be compressed after 1 day and deleted "
"after a bigger number of days."
)
def _clean_v3bw_files(args, conf):
v3bw_dname = conf.getpath('paths', 'v3bw_dname')
v3bw_dname = conf.getpath("paths", "v3bw_dname")
if not os.path.isdir(v3bw_dname):
fail_hard('%s does not exist', v3bw_dname)
compress_after_days = conf.getint('cleanup',
'v3bw_files_compress_after_days')
delete_after_days = conf.getint('cleanup',
'v3bw_files_delete_after_days')
fail_hard("%s does not exist", v3bw_dname)
compress_after_days = conf.getint(
"cleanup", "v3bw_files_compress_after_days"
)
delete_after_days = conf.getint("cleanup", "v3bw_files_delete_after_days")
_check_validity_periods_v3bw(compress_after_days, delete_after_days)
# first delete so that the files to be deleted are not compressed first
files_to_delete = _get_files_mtime_older_than(v3bw_dname,
delete_after_days,
['.v3bw', '.gz'])
files_to_delete = _get_files_mtime_older_than(
v3bw_dname, delete_after_days, [".v3bw", ".gz"]
)
_delete_files(v3bw_dname, files_to_delete, dry_run=args.dry_run)
files_to_compress = _get_files_mtime_older_than(v3bw_dname,
compress_after_days,
['.v3bw'])
files_to_compress = _get_files_mtime_older_than(
v3bw_dname, compress_after_days, [".v3bw"]
)
# when dry_run is true, compress will also show all the files that
# would have been deleted, since they are not really deleted
_compress_files(v3bw_dname, files_to_compress, dry_run=args.dry_run)
def _clean_result_files(args, conf):
datadir = conf.getpath('paths', 'datadir')
datadir = conf.getpath("paths", "datadir")
if not os.path.isdir(datadir):
fail_hard('%s does not exist', datadir)
fail_hard("%s does not exist", datadir)
compress_after_days = conf.getint(
'cleanup', 'data_files_compress_after_days')
delete_after_days = conf.getint(
'cleanup', 'data_files_delete_after_days')
"cleanup", "data_files_compress_after_days"
)
delete_after_days = conf.getint("cleanup", "data_files_delete_after_days")
# first delete so that the files to be deleted are not compressed first
files_to_delete = _get_files_mtime_older_than(
datadir, delete_after_days, ['.txt', '.gz'])
datadir, delete_after_days, [".txt", ".gz"]
)
_delete_files(datadir, files_to_delete, dry_run=args.dry_run)
# when dry_run is true, compress will also show all the files that
# would have been deleted, since they are not really deleted
files_to_compress = _get_files_mtime_older_than(
datadir, compress_after_days, ['.txt'])
datadir, compress_after_days, [".txt"]
)
_compress_files(datadir, files_to_compress, dry_run=args.dry_run)
def main(args, conf):
'''
"""
Main entry point in to the cleanup command.
:param argparse.Namespace args: command line arguments
:param configparser.ConfigParser conf: parsed config files
'''
"""
if not args.no_results:
_clean_result_files(args, conf)
......
from math import ceil
from sbws.globals import (fail_hard, SBWS_SCALE_CONSTANT, TORFLOW_SCALING,
SBWS_SCALING, TORFLOW_BW_MARGIN, PROP276_ROUND_DIG,
DAY_SECS, NUM_MIN_RESULTS, GENERATE_PERIOD)
from sbws.globals import (
fail_hard,
SBWS_SCALE_CONSTANT,
TORFLOW_SCALING,
SBWS_SCALING,
TORFLOW_BW_MARGIN,
PROP276_ROUND_DIG,
DAY_SECS,
NUM_MIN_RESULTS,
GENERATE_PERIOD,
)
from sbws.lib.v3bwfile import V3BWFile
from sbws.lib.resultdump import load_recent_results_in_datadir
from argparse import ArgumentDefaultsHelpFormatter
......@@ -15,72 +23,119 @@ log = logging.getLogger(__name__)
def gen_parser(sub):
d = 'Generate a v3bw file based on recent results. A v3bw file is the '\
'file Tor directory authorities want to read and base their '\
'bandwidth votes on. '\
'To avoid inconsistent reads, configure tor with '\
'"V3BandwidthsFile /path/to/latest.v3bw". '\
'(latest.v3bw is an atomically created symlink in the same '\
'directory as output.) '\
'If the file is transferred to another host, it should be written to '\
'a temporary path, then renamed to the V3BandwidthsFile path.\n'\
'The default scaling method is torflow\'s one. To use different'\
'scaling methods or no scaling, see the options.'
p = sub.add_parser('generate', description=d,
formatter_class=ArgumentDefaultsHelpFormatter)
p.add_argument('--output', default=None, type=str,
help='If specified, write the v3bw here instead of what is'
'specified in the configuration')
d = (
"Generate a v3bw file based on recent results. A v3bw file is the "
"file Tor directory authorities want to read and base their "
"bandwidth votes on. "
"To avoid inconsistent reads, configure tor with "
'"V3BandwidthsFile /path/to/latest.v3bw". '
"(latest.v3bw is an atomically created symlink in the same "
"directory as output.) "
"If the file is transferred to another host, it should be written to "
"a temporary path, then renamed to the V3BandwidthsFile path.\n"
"The default scaling method is torflow's one. To use different"
"scaling methods or no scaling, see the options."
)
p = sub.add_parser(
"generate",
description=d,
formatter_class=ArgumentDefaultsHelpFormatter,
)
p.add_argument(
"--output",
default=None,
type=str,
help="If specified, write the v3bw here instead of what is"
"specified in the configuration",
)
# The reason for --scale-constant defaulting to 7500 is because at one
# time, torflow happened to generate output that averaged to 7500 bw units
# per relay. We wanted the ability to try to be like torflow. See
# https://lists.torproject.org/pipermail/tor-dev/2018-March/013049.html
p.add_argument('--scale-constant', default=SBWS_SCALE_CONSTANT, type=int,
help='When scaling bw weights, scale them using this const '
'multiplied by the number of measured relays')
p.add_argument('--scale-sbws', action='store_true',
help='If specified, do not use bandwidth values as they '
'are, but scale them such that we have a budget of '
'scale_constant * num_measured_relays = bandwidth to give '
'out, and we do so proportionally')
p.add_argument('-t', '--scale-torflow', action='store_true',
default=True,
help='If specified, scale measurements using torflow\'s '
'method. This option is kept for compatibility with older '
'versions and it is silently ignored, since it is the '
'default.')
p.add_argument('-w', '--raw', action='store_true',
help='If specified, do use bandwidth raw measurements '
'without any scaling.')
p.add_argument('-m', '--torflow-bw-margin', default=TORFLOW_BW_MARGIN,
type=float,
help="Cap maximum bw when scaling as Torflow. ")