GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Commit 5b6ba3e8 authored by juga  's avatar juga 💬

fix: relaylist: Count recent relay's monitoring numbers

using timestamps class.

Additionally:

- fix: relayprioritizer: Replace call relay priority

- fix: scanner: Replace call relay measurement attempt
parent c02d9b1e
......@@ -516,7 +516,7 @@ def main_loop(args, conf, controller, relay_list, circuit_builder, result_dump,
if settings.end_event.is_set():
break
relay_list.increment_recent_measurement_attempt()
target.increment_relay_recent_measurement_attempt_count()
target.increment_relay_recent_measurement_attempt()
num_relays += 1
# callback and callback_err must be non-blocking
callback = result_putter(result_dump)
......
......@@ -11,6 +11,7 @@ from threading import Lock
from ..globals import (
MAX_RECENT_CONSENSUS_COUNT,
MAX_RECENT_PRIORITY_RELAY_COUNT,
MAX_RECENT_PRIORITY_LIST_COUNT,
MEASUREMENTS_PERIOD
)
from ..util import timestamp, timestamps
......@@ -18,24 +19,6 @@ from ..util import timestamp, timestamps
log = logging.getLogger(__name__)
def remove_old_consensus_timestamps(
consensus_timestamps, measurements_period=MEASUREMENTS_PERIOD):
"""
Remove the consensus timestamps that are older than period for which
the measurements are keep from a list of consensus_timestamps.
:param list consensus_timestamps:
:param int measurements_period:
:returns list: a new list of ``consensus_timestamps``
"""
new_consensus_timestamps = [
t
for t in consensus_timestamps
if not timestamp.is_old(t, measurements_period)
]
return new_consensus_timestamps
def valid_after_from_network_statuses(network_statuses):
"""Obtain the consensus Valid-After datetime from the ``document``
attribute of a ``stem.descriptor.RouterStatusEntryV3``.
......@@ -84,14 +67,20 @@ class Relay:
self._desc = cont.get_server_descriptor(fp, default=None)
except (DescriptorUnavailable, ControllerError) as e:
log.exception("Exception trying to get desc %s", e)
self._consensus_timestamps = []
self._add_consensus_timestamp(timestamp)
self.relay_in_recent_consensus = timestamps.DateTimeSeq(
[], MAX_RECENT_CONSENSUS_COUNT
)
self.update_relay_in_recent_consensus()
# The number of times that a relay is "prioritized" to be measured.
# It is incremented in ``RelayPrioritizer.best_priority``
self.relay_recent_priority_list_count = 0
self.relay_recent_priority_list = timestamps.DateTimeSeq(
[], MAX_RECENT_PRIORITY_LIST_COUNT
)
# The number of times that a relay has been queued to be measured.
# It is incremented in ``scanner.main_loop``
self.relay_recent_measurement_attempt_count = 0
self.relay_recent_measurement_attempt = timestamps.DateTimeSeq(
[], MAX_RECENT_PRIORITY_LIST_COUNT
)
def _from_desc(self, attr):
if not self._desc:
......@@ -178,68 +167,15 @@ class Relay:
@property
def last_consensus_timestamp(self):
if len(self._consensus_timestamps) >= 1:
return self._consensus_timestamps[-1]
return None
return self.relay_in_recent_consensus.last()
def _append_consensus_timestamp_if_later(self, timestamp):
"""Append timestamp to the list of consensus timestamps, if it is later
than the most recent existing timestamp, or there are no timestamps.
Should only be called by _add_consensus_timestamp().
timestamp must not be None, and it must not be zero.
"""
if not timestamp:
log.info('Bad timestamp %s, skipping consensus timestamp '
'update for relay %s', timestamp, self.fingerprint)
return
# The consensus timestamp list was initialized.
if self.last_consensus_timestamp is not None:
# timestamp is more recent than the most recent stored
# consensus timestamp.
if timestamp > self.last_consensus_timestamp:
# Add timestamp
self._consensus_timestamps.append(timestamp)
# The consensus timestamp list was not initialized.
else:
# Add timestamp
self._consensus_timestamps.append(timestamp)
def _add_consensus_timestamp(self, timestamp=None):
"""Add the consensus timestamp in which this relay is present.
"""
# It is possible to access to the relay's consensensus Valid-After
# so believe it, rather than the supplied timestamp
if self.consensus_valid_after is not None:
self._append_consensus_timestamp_if_later(
self.consensus_valid_after
)
elif timestamp:
# Add the arg timestamp.
self._append_consensus_timestamp_if_later(timestamp)
# In any other case
else:
log.warning('Bad timestamp %s, using current time for consensus '
'timestamp update for relay %s',
timestamp, self.fingerprint)
# Add the current datetime
self._append_consensus_timestamp_if_later(
datetime.utcnow().replace(microsecond=0))
def _remove_old_consensus_timestamps(
self, measurements_period=MEASUREMENTS_PERIOD):
self._consensus_timestamps = \
remove_old_consensus_timestamps(
copy.deepcopy(self._consensus_timestamps), measurements_period
)
def update_consensus_timestamps(self, timestamp=None):
self._add_consensus_timestamp(timestamp)
self._remove_old_consensus_timestamps()
def update_relay_in_recent_consensus(self, timestamp=None):
self.relay_in_recent_consensus.update(timestamp)
@property
def relay_in_recent_consensus_count(self):
"""Number of times the relay was in a conensus."""
return len(self._consensus_timestamps)
return len(self.relay_in_recent_consensus)
def can_exit_to_port(self, port):
"""
......@@ -271,19 +207,20 @@ class Relay:
Flag.EXIT in self.flags and
self.can_exit_to_port(port))
def increment_relay_recent_measurement_attempt_count(self):
def increment_relay_recent_measurement_attempt(self):
"""
Increment The number of times that a relay has been queued
to be measured.
It is call from :funf:`~sbws.core.scaner.main_loop`.
"""
# If it was not in the previous measurements version, start counting
if self.relay_recent_measurement_attempt_count is None:
self.relay_recent_measurement_attempt_count = 0
self.relay_recent_measurement_attempt_count += 1
self.relay_recent_measurement_attempt.update()
def increment_relay_recent_priority_list_count(self):
@property
def relay_recent_measurement_attempt_count(self):
return len(self.relay_recent_measurement_attempt)
def increment_relay_recent_priority_list(self):
"""
The number of times that a relay is "prioritized" to be measured.
......@@ -291,9 +228,11 @@ class Relay:
:meth:`~sbws.lib.relayprioritizer.RelayPrioritizer.best_priority`.
"""
# If it was not in the previous measurements version, start counting
if self.relay_recent_priority_list_count is None:
self.relay_recent_priority_list_count = 0
self.relay_recent_priority_list_count += 1
self.relay_recent_priority_list.update()
@property
def relay_recent_priority_list_count(self):
return len(self.relay_recent_priority_list)
def is_old(self):
"""Whether the last consensus seen for this relay is older than the
......@@ -447,7 +386,7 @@ class RelayList:
fp = r.fingerprint
# new_relays_dict[fp] is the router status.
r.update_router_status(new_relays_dict[fp])
r.update_consensus_timestamps(timestamp)
r.update_relay_in_recent_consensus(timestamp)
try:
descriptor = c.get_server_descriptor(fp, default=None)
except (DescriptorUnavailable, ControllerError) as e:
......
......@@ -164,5 +164,5 @@ class RelayPrioritizer:
del(relay.priority)
# Increment the number of times a realy was "prioritized" to be
# measured.
relay.increment_relay_recent_priority_list_count()
relay.increment_relay_recent_priority_list()
yield relay
......@@ -93,6 +93,28 @@ def controller_5days_later(router_statuses_5days_later):
return controller
@pytest.fixture(scope="session")
def server_descriptors(root_data_path):
p = os.path.join(root_data_path, "2020-02-29-10-05-00-server-descriptors")
server_descriptors = descriptor.parse_file(p)
server_descriptors_list = list(server_descriptors)
return server_descriptors_list
@pytest.fixture(scope="session")
def server_descriptor(server_descriptors):
return server_descriptors[0]
@pytest.fixture(scope="session")
def router_status(server_descriptor, router_statuses):
rs = [
ns
for ns in router_statuses
if ns.fingerprint == server_descriptor.fingerprint
][0]
return rs
# Because of the function scoped `args` in `tests.unit.conftest`, this has to
# be function scoped too.
@pytest.fixture(scope='function')
......
This diff is collapsed.
"""relaylist.py unit tests."""
from datetime import datetime, timedelta
from datetime import datetime
# When datetime is imported as a class (`from datetime import datetime`) it can
# not be mocked because it is a built-in type. It can only be mocked when
......@@ -7,23 +7,10 @@ from datetime import datetime, timedelta
# freezegun is able to mock any datetime object, it also allows comparations.
from freezegun import freeze_time
from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps
from sbws.lib.relaylist import Relay, RelayList
from sbws.util.state import State
def test_remove_old_consensus_timestamps():
days_ago = datetime(2020, 3, 1)
timestamps = [days_ago] + [
days_ago + timedelta(days=x) for x in range(1, 5)
]
with freeze_time(days_ago + timedelta(days=5, seconds=1)):
new_timestamps = remove_old_consensus_timestamps(
timestamps, 5 * 24 * 60 * 60
)
assert len(new_timestamps) == len(timestamps) - 1
assert days_ago not in new_timestamps
def test_init_relays(
args, conf, controller, controller_1h_later, controller_5days_later
):
......@@ -40,7 +27,7 @@ def test_init_relays(
with freeze_time("2020-02-29 10:00:00"):
relay_list = RelayList(args, conf, controller, state=state)
assert relay_list.recent_consensus_count == 1
assert len(relay_list._relays[0]._consensus_timestamps) == 1
assert relay_list._relays[0].relay_in_recent_consensus_count == 1
# The actual number of relays in the consensus
assert len(relay_list._relays) == 6433
fps = {r.fingerprint for r in relay_list._relays}
......@@ -51,7 +38,7 @@ def test_init_relays(
# Call relays update the list of relays.
relay_list.relays
assert relay_list.recent_consensus_count == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2
assert relay_list._relays[0].relay_in_recent_consensus_count == 2
# Check that the number of relays is now the previous one plus the relays
# that are in the new consensus that there were not in the previous one.
fps_1h_later = {r.fingerprint for r in relay_list._relays}
......@@ -64,7 +51,7 @@ def test_init_relays(
with freeze_time("2020-03-05 10:00:01"):
relay_list.relays
assert relay_list.recent_consensus_count == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2
assert relay_list._relays[0].relay_in_recent_consensus_count == 2
fps_5days_later = {r.fingerprint for r in relay_list._relays}
# The number of added relays will be the number of relays in this
# consensus that were not in the other 2 conensuses
......@@ -117,3 +104,76 @@ def test_increment_recent_measurement_attempt(args, conf, controller):
relay_list.increment_recent_measurement_attempt()
assert 3 == relay_list.recent_measurement_attempt_count
assert 3 == len(state["recent_measurement_attempt"])
def test_increment_relay_recent_measurement_attempt(
controller, router_status, server_descriptor
):
"""Test that incrementing the measurement attempts do not go on forever
And instead it only counts the number of attempts in the last days.
"""
# For this test it does not matter that the consensus timestamps
# are not correct.
now = datetime.utcnow()
relay = Relay("A" * 40, controller, timestamp=now)
# The initial count is 0 and the state does not have that key.
assert 0 == relay.relay_recent_measurement_attempt_count
# Pretend that a measurement attempt is made.
with freeze_time("2020-02-29 10:00:00"):
relay.increment_relay_recent_measurement_attempt()
assert 1 == relay.relay_recent_measurement_attempt_count
# And a second measurement attempt is made 4 days later.
with freeze_time("2020-03-04 10:00:00"):
relay.increment_relay_recent_measurement_attempt()
assert 2 == relay.relay_recent_measurement_attempt_count
# And a third measurement attempt is made 5 days later.
with freeze_time("2020-03-05 10:00:00"):
relay.increment_relay_recent_measurement_attempt()
assert 3 == relay.relay_recent_measurement_attempt_count
# And a fourth measurement attempt is made 6 days later. The first one is
# now removed and not counted.
with freeze_time("2020-03-06 10:00:00"):
relay.increment_relay_recent_measurement_attempt()
assert 3 == relay.relay_recent_measurement_attempt_count
# XXX: Write to a Result and load it back
def test_increment_relay_recent_priority_list(
controller, router_status, server_descriptor
):
"""Test that incrementing the priority lists do not go on forever
And instead it only counts the number of priority lists in the last days.
"""
# For this test it does not matter that the consensus timestamps
# are not correct.
now = datetime.utcnow()
relay = Relay("A" * 40, controller, timestamp=now)
# The initial count is 0 and the state does not have that key.
assert 0 == relay.relay_recent_priority_list_count
# Pretend that a measurement attempt is made.
with freeze_time("2020-02-29 10:00:00"):
relay.increment_relay_recent_priority_list()
assert 1 == relay.relay_recent_priority_list_count
# And a second measurement attempt is made 4 days later.
with freeze_time("2020-03-04 10:00:00"):
relay.increment_relay_recent_priority_list()
assert 2 == relay.relay_recent_priority_list_count
# And a third measurement attempt is made 5 days later.
with freeze_time("2020-03-05 10:00:00"):
relay.increment_relay_recent_priority_list()
assert 3 == relay.relay_recent_priority_list_count
# And a fourth measurement attempt is made 6 days later. The first one is
# now removed and not counted.
with freeze_time("2020-03-06 10:00:00"):
relay.increment_relay_recent_priority_list()
assert 3 == relay.relay_recent_priority_list_count
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment