GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Commit e361caf8 authored by juga  's avatar juga 💬

Merge branch 'maint-1.1_bug33570_consensus_ts' into maint-1.1

parents 55302251 b3a84ee1
......@@ -25,6 +25,8 @@ by a :term:`directory authority` to report relays’ bandwidth in its vote.
:height: 200px
:align: center
.. image:: ./images/dirauths_bwauths.png
Intialization
~~~~~~~~~~~~~~
......
......@@ -24,9 +24,11 @@ def remove_old_consensus_timestamps(
:param int measurements_period:
:returns list: a new list of ``consensus_timestamps``
"""
oldest_date = datetime.utcnow() - timedelta(measurements_period)
new_consensus_timestamps = \
[t for t in consensus_timestamps if t >= oldest_date]
new_consensus_timestamps = [
t
for t in consensus_timestamps
if not timestamp.is_old(t, measurements_period)
]
return new_consensus_timestamps
......@@ -454,9 +456,9 @@ class RelayList:
# If a relay in the previous consensus and is in the current
# one, update its timestamp, router status and descriptor.
fp = r.fingerprint
r.update_consensus_timestamps(timestamp)
# new_relays_dict[fp] is the router status.
r.update_router_status(new_relays_dict[fp])
r.update_consensus_timestamps(timestamp)
try:
descriptor = c.get_server_descriptor(fp, default=None)
except (DescriptorUnavailable, ControllerError) as e:
......@@ -471,7 +473,7 @@ class RelayList:
# If the relay is not in the current consensus but is not "old"
# yet, add it to the new list of relays too, though its timestamp,
# router status and descriptor can't be updated.
elif not r.is_old(self._measurements_period):
elif not r.is_old():
new_relays.append(r)
# Otherwise, don't add it to the new list of relays.
# For debugging, count the old relays that will be discarded.
......@@ -483,11 +485,12 @@ class RelayList:
r = Relay(ns.fingerprint, c, ns=ns, timestamp=timestamp)
new_relays.append(r)
days = self._measurements_period / (60 * 60 * 24)
log.debug("Previous number of relays being measured %d",
len(self._relays))
log.debug("Number of relays not in the in the consensus in the last "
"%d days: %d.",
self._measurements_period, num_old_relays)
days, num_old_relays)
log.debug("Number of relays to measure with the current consensus: "
"%d", len(new_relays))
return new_relays
......
......@@ -86,9 +86,9 @@ def is_old(timestamp, measurements_period=MEASUREMENTS_PERIOD):
# This will raise an exception if the string is not correctly
# formatted.
timestamp = isostr_to_dt_obj(timestamp)
else:
elif isinstance(timestamp, int) or isinstance(timestamp, float):
# This will raise an exception if the type is not int or float or
# is not actually a timestamp
timestamp = unixts_to_dt_obj(timestamp)
oldest_date = datetime.utcnow() - timedelta(measurements_period)
return timestamp > oldest_date
oldest_date = datetime.utcnow() - timedelta(seconds=measurements_period)
return timestamp < oldest_date
......@@ -75,7 +75,7 @@ setup(
extras_require={
# vulture: find unused code
'dev': ['flake8', 'vulture'],
'test': ['tox', 'pytest', 'coverage'],
'test': ['tox', 'pytest', 'coverage', 'freezegun'],
# recommonmark: to make sphinx render markdown
'doc': ['sphinx', 'recommonmark', 'pylint'],
},
......
"""Common pytest configuration for unit and integration tests."""
import pytest
import os.path
from unittest import mock
from stem import descriptor
from sbws.util.parser import create_parser
......@@ -10,7 +15,7 @@ def parser():
@pytest.fixture()
def datadir(request):
"""get, read, open test files from the tests "data" directory."""
"""get, read, open test files from the tests relative "data" directory."""
class D:
def __init__(self, basepath):
self.basepath = basepath
......@@ -29,3 +34,56 @@ def datadir(request):
with self.open(name, "r") as f:
return f.readlines()
return D(request.fspath.dirpath("data"))
@pytest.fixture(scope="session")
def root_data_path():
"""Path to the data dir in the tests root, for both unit and integration
tests.
"""
return os.path.join(os.path.dirname(os.path.abspath(__file__)), "data",)
@pytest.fixture(scope="session")
def router_statuses(root_data_path):
p = os.path.join(root_data_path, "2020-02-29-10-00-00-consensus")
network_statuses = descriptor.parse_file(p)
network_statuses_list = list(network_statuses)
return network_statuses_list
@pytest.fixture(scope="session")
def router_statuses_1h_later(root_data_path):
p = os.path.join(root_data_path, "2020-02-29-11-00-00-consensus")
network_statuses = descriptor.parse_file(p)
network_statuses_list = list(network_statuses)
return network_statuses_list
@pytest.fixture(scope="session")
def router_statuses_5days_later(root_data_path):
p = os.path.join(root_data_path, "2020-03-05-10-00-00-consensus")
network_statuses = descriptor.parse_file(p)
network_statuses_list = list(network_statuses)
return network_statuses_list
@pytest.fixture(scope="session")
def controller(router_statuses):
controller = mock.Mock()
controller.get_network_statuses.return_value = router_statuses
return controller
@pytest.fixture(scope="session")
def controller_1h_later(router_statuses_1h_later):
controller = mock.Mock()
controller.get_network_statuses.return_value = router_statuses_1h_later
return controller
@pytest.fixture(scope="session")
def controller_5days_later(router_statuses_5days_later):
controller = mock.Mock()
controller.get_network_statuses.return_value = router_statuses_5days_later
return controller
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
"""relaylist.py unit tests."""
from datetime import datetime, timedelta
# When datetime is imported as a class (`from datetime import datetime`) it can
# not be mocked because it is a built-in type. It can only be mocked when
# imported as module.
# freezegun is able to mock any datetime object, it also allows comparations.
from freezegun import freeze_time
from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps
def test_remove_old_consensus_timestamps():
days_ago = datetime(2020, 3, 1)
timestamps = [days_ago] + [
days_ago + timedelta(days=x) for x in range(1, 5)
]
with freeze_time(days_ago + timedelta(days=5, seconds=1)):
new_timestamps = remove_old_consensus_timestamps(
timestamps, 5 * 24 * 60 * 60
)
assert len(new_timestamps) == len(timestamps) - 1
assert days_ago not in new_timestamps
def test_init_relays(
args, conf, controller, controller_1h_later, controller_5days_later
):
"""
Test `init_relays` when creating the RelayList the first time and when a
new consensus is received.
Test that the number of consesus timesamps and relays is correct.
"""
# There is no need to mock datetime to update the consensus, since the
# actual date will be always later.
# But it's needed to have the correct list of timestamps both for RelayList
# and Relay.
with freeze_time("2020-02-29 10:00:00"):
relay_list = RelayList(args, conf, controller=controller)
assert len(relay_list._consensus_timestamps) == 1
assert len(relay_list._relays[0]._consensus_timestamps) == 1
# The actual number of relays in the consensus
assert len(relay_list._relays) == 6433
fps = {r.fingerprint for r in relay_list._relays}
# One hour later there is a new consensus
relay_list._controller = controller_1h_later
with freeze_time("2020-02-29 11:00:00"):
# Call relays update the list of relays.
relay_list.relays
assert len(relay_list._consensus_timestamps) == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2
# Check that the number of relays is now the previous one plus the relays
# that are in the new consensus that there were not in the previous one.
fps_1h_later = {r.fingerprint for r in relay_list._relays}
added_fps = fps_1h_later.difference(fps)
assert 6505 == 6433 + len(added_fps)
# Five days later plus 1 second.
# The first consensus timestamp will get removed.
relay_list._controller = controller_5days_later
with freeze_time("2020-03-05 10:00:01"):
relay_list.relays
assert len(relay_list._consensus_timestamps) == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2
fps_5days_later = {r.fingerprint for r in relay_list._relays}
# The number of added relays will be the number of relays in this
# consensus that were not in the other 2 conensuses
added_fps = fps_5days_later.difference(fps_1h_later)
# The number of removed relays that are in this consensus, plus the added
# ones that were not in the first consensus (because it has been removed).
removed_fps = fps.difference(fps_5days_later)
# The number of relays will be the number of relays in the cosensus plus
# the added ones minus the removed ones.
assert 6925 == 6505 + len(added_fps) - len(removed_fps)
# -*- coding: utf-8 -*-
"""Test timestamp conversion util functions"""
from datetime import datetime, timezone
from datetime import datetime, timezone, timedelta
from sbws.util.timestamp import (dt_obj_to_isodt_str, unixts_to_dt_obj,
unixts_to_isodt_str, unixts_to_str)
unixts_to_isodt_str, unixts_to_str, is_old)
isodt_str = '2018-05-23T12:55:04'
......@@ -25,3 +24,14 @@ def test_unixts_to_isodt_str():
def test_unixts_to_str():
assert str(unixts) == unixts_to_str(unixts)
def test_is_old():
# Since this timestamp is generated a few microseconds before checking
# the oldest timestamp, it will be old.
old_timestamp = datetime.utcnow() - timedelta(days=5)
assert is_old(old_timestamp)
# A recent timestamp should be at least 1 second newer that the oldest
recent_timestamp = datetime.utcnow() - timedelta(days=5) \
+ timedelta(seconds=1)
assert not is_old(recent_timestamp)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment