Commit e92645fb authored by Matt Traudt's avatar Matt Traudt
Browse files

Switch to UTC

- Also do some switching away from time_now() back to time.time()
GH: ref #67

- Add note in CONTRIBUTING to prefer UTC and timestamps

- Fix things that broke when switching from date to datetime.

GH: closes #113
parent aba21b92
......@@ -36,6 +36,17 @@ commits that show up in ``git blame`` 10 years from now make more sense.
Simple Bandwidth Scanner is moving towards 100% test coverage. Please help us
reach that goal, or at least not move us away from it.
Coding Guidelines
=================
**Strongly prefer Unix timestamps** over ``datetime`` structures and always
work in UTC for as long as possible. When reading/writing/manipulating results
from some period of time in the past, always err on the side of caution. For
example, open an extra file into the past just in case it happens to include
result lines that have timestamps that are still considered valid (of course,
ignore results in the file that are no longer valid).
.. _commit-msg:
Example commit message
......
......@@ -6,7 +6,7 @@ from sbws.lib.resultdump import load_recent_results_in_datadir
from sbws.lib.resultdump import group_results_by_relay
from argparse import ArgumentDefaultsHelpFormatter
import os
from datetime import date
from datetime import datetime
from datetime import timedelta
from statistics import mean
import logging
......@@ -80,8 +80,8 @@ def print_stats(args, data):
fastest_transfers[0]
first_time = min([r.time for r in results])
last_time = max([r.time for r in results])
first = date.fromtimestamp(first_time)
last = date.fromtimestamp(last_time)
first = datetime.utcfromtimestamp(first_time)
last = datetime.utcfromtimestamp(last_time)
duration = timedelta(seconds=last_time-first_time)
# remove microseconds for prettier printing
duration = duration - timedelta(microseconds=duration.microseconds)
......
from sbws.globals import time_now
import os
import json
import time
import logging
from glob import glob
from threading import Thread
......@@ -8,7 +8,7 @@ from threading import Event
from threading import RLock
from queue import Queue
from queue import Empty
from datetime import date
from datetime import datetime
from datetime import timedelta
from enum import Enum
from stem.descriptor.router_status_entry import RouterStatusEntryV3
......@@ -65,7 +65,7 @@ def trim_results(fresh_days, results):
assert isinstance(fresh_days, int)
assert isinstance(results, list)
data_period = fresh_days * 24*60*60
oldest_allowed = time_now() - data_period
oldest_allowed = time.time() - data_period
out_results = []
for result in results:
if result.time >= oldest_allowed:
......@@ -81,7 +81,7 @@ def load_recent_results_in_datadir(fresh_days, datadir, success_only=False):
assert isinstance(fresh_days, int)
assert os.path.isdir(datadir)
results = []
today = date.fromtimestamp(time_now())
today = datetime.utcfromtimestamp(time.time())
data_period = fresh_days + 2
oldest_day = today - timedelta(days=data_period)
working_day = oldest_day
......@@ -89,8 +89,9 @@ def load_recent_results_in_datadir(fresh_days, datadir, success_only=False):
# Cannot use ** and recursive=True in glob() because we support 3.4
# So instead settle on finding files in the datadir and one
# subdirectory below the datadir that fit the form of YYYY-MM-DD*.txt
patterns = [os.path.join(datadir, '{}*.txt'.format(working_day)),
os.path.join(datadir, '*', '{}*.txt'.format(working_day))]
d = working_day.date()
patterns = [os.path.join(datadir, '{}*.txt'.format(d)),
os.path.join(datadir, '*', '{}*.txt'.format(d))]
for pattern in patterns:
for fname in glob(pattern):
results.extend(load_result_file(
......@@ -110,11 +111,12 @@ def write_result_to_datadir(result, datadir):
''' Can be called from any thread '''
assert isinstance(result, Result)
assert os.path.isdir(datadir)
dt = date.fromtimestamp(result.time)
dt = datetime.utcfromtimestamp(result.time)
ext = '.txt'
result_fname = os.path.join(
datadir, '{}{}'.format(dt, ext))
datadir, '{}{}'.format(dt.date(), ext))
with DirectoryLock(datadir):
log.debug('Writing a result to %s', result_fname)
with open(result_fname, 'at') as fd:
fd.write('{}\n'.format(str(result)))
......@@ -149,7 +151,7 @@ class Result:
self._circ = circ
self._server_host = server_host
self._scanner = scanner_nick
self._time = time_now() if t is None else t
self._time = time.time() if t is None else t
@property
def type(self):
......
......@@ -6,7 +6,9 @@ from sbws.lib.resultdump import Result
from sbws.lib.resultdump import write_result_to_datadir
import sbws.core.init
import sbws.core.stats
from datetime import date
from tests.globals import monotonic_time
from unittest.mock import patch
from datetime import datetime
import os
import time
import logging
......@@ -39,16 +41,16 @@ def add_single_fresh_result(dname):
write_result_to_datadir(r, dd)
def add_two_fresh_results(dname):
def add_two_fresh_results(dname, t):
r1 = ResultError(
Result.Relay('DEADBEEF1111', 'CowSayWhat', '127.0.0.1'),
['DEADBEEF1111', 'BEADDEEF2222'],
'127.0.1.1', 'SBWSscanner', t=time.time())
'127.0.1.1', 'SBWSscanner', t=t)
r2 = ResultSuccess(
[1, 2, 3], [{'amount': 100, 'duration': 1}],
Result.Relay('DEADBEEF1111', 'CowSayWhat', '127.0.0.1'),
['DEADBEEF1111', 'BEADDEEF2222'],
'127.0.1.1', 'SBWSscanner', t=time.time())
'127.0.1.1', 'SBWSscanner', t=t)
dd = os.path.join(str(dname), 'datadir')
os.makedirs(dd)
write_result_to_datadir(r1, dd)
......@@ -130,21 +132,24 @@ def test_stats_fresh_result(tmpdir, capsys, caplog):
lines = [l.getMessage() for l in caplog.records]
needed_log_lines = [
'Read 1 lines from {}/{}/{}.txt'.format(
tmpdir, 'datadir', date.fromtimestamp(time.time())),
tmpdir, 'datadir', datetime.utcfromtimestamp(time.time()).date()),
'Keeping 1/1 results',
]
for needed_line in needed_log_lines:
assert needed_line in lines
def test_stats_fresh_results(tmpdir, capsys, caplog):
@patch('time.time')
def test_stats_fresh_results(time_mock, tmpdir, capsys, caplog):
'''
An initialized .sbws directory with a fresh error and fresh success should
have some exciting stats and exit cleanly
'''
caplog.set_level(logging.DEBUG)
init_directory(tmpdir)
add_two_fresh_results(tmpdir)
start = 1524769441
time_mock.side_effect = monotonic_time(start=start)
add_two_fresh_results(tmpdir, start-1)
p = create_parser()
args = p.parse_args(
'-d {} --log-level DEBUG stats --error-types'.format(tmpdir).split())
......@@ -163,7 +168,7 @@ def test_stats_fresh_results(tmpdir, capsys, caplog):
lines = [l.getMessage() for l in caplog.records]
needed_log_lines = [
'Read 2 lines from {}/{}/{}.txt'.format(
tmpdir, 'datadir', date.fromtimestamp(time.time())),
tmpdir, 'datadir', datetime.utcfromtimestamp(time.time()).date()),
'Keeping 2/2 results',
'Found a _ResultType.Error for the first time',
'Found a _ResultType.Success for the first time',
......
......@@ -10,13 +10,13 @@ from sbws.lib.resultdump import _ResultType
from tests.globals import monotonic_time
@patch('sbws.lib.resultdump.time_now')
def test_Result(time_now):
@patch('time.time')
def test_Result(time_mock):
'''
A standard Result should not be convertible to a string because Result.type
is not implemented.
'''
time_now.side_effect = monotonic_time()
time_mock.side_effect = monotonic_time()
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -58,10 +58,10 @@ def test_Result_from_dict_bad_type():
assert None, 'Should have failed'
@patch('sbws.lib.resultdump.time_now')
def test_ResultSuccess(time_now):
@patch('time.time')
def test_ResultSuccess(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -89,10 +89,10 @@ def test_ResultSuccess(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultSuccess_from_dict(time_now):
@patch('time.time')
def test_ResultSuccess_from_dict(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -116,10 +116,10 @@ def test_ResultSuccess_from_dict(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultError(time_now):
@patch('time.time')
def test_ResultError(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -144,10 +144,10 @@ def test_ResultError(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultError_from_dict(time_now):
@patch('time.time')
def test_ResultError_from_dict(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -170,10 +170,10 @@ def test_ResultError_from_dict(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultErrorCircuit(time_now):
@patch('time.time')
def test_ResultErrorCircuit(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -199,10 +199,10 @@ def test_ResultErrorCircuit(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultErrorCircuit_from_dict(time_now):
@patch('time.time')
def test_ResultErrorCircuit_from_dict(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -225,10 +225,10 @@ def test_ResultErrorCircuit_from_dict(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultErrorStream(time_now):
@patch('time.time')
def test_ResultErrorStream(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -254,10 +254,10 @@ def test_ResultErrorStream(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultErrorStream_from_dict(time_now):
@patch('time.time')
def test_ResultErrorStream_from_dict(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -280,10 +280,10 @@ def test_ResultErrorStream_from_dict(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultErrorAuth(time_now):
@patch('time.time')
def test_ResultErrorAuth(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......@@ -309,10 +309,10 @@ def test_ResultErrorAuth(time_now):
assert str(r1) == str(r2)
@patch('sbws.lib.resultdump.time_now')
def test_ResultErrorAuth_from_dict(time_now):
@patch('time.time')
def test_ResultErrorAuth_from_dict(time_mock):
t = 2000
time_now.side_effect = monotonic_time(start=t)
time_mock.side_effect = monotonic_time(start=t)
fp1 = 'A' * 40
fp2 = 'Z' * 40
circ = [fp1, fp2]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment