Merge branch 'bug_33009_v6' into 'maint-1.1'

Bug 33009 v6

See merge request torproject/network-health/sbws!6

sbws/core/scanner.py

@@ -211,10 +211,16 @@ def _pick_ideal_second_hop(relay, dest, rl, cont, is_exit):
         else rl.non_exits
     if not len(candidates):
         return None
+    min_relay_bw = rl.exit_min_bw() if is_exit else rl.non_exit_min_bw()
     log.debug('Picking a 2nd hop to measure %s from %d choices. is_exit=%s',
               relay.nickname, len(candidates), is_exit)
     for min_bw_factor in [2, 1.75, 1.5, 1.25, 1]:
         min_bw = relay.consensus_bandwidth * min_bw_factor
+        # We might have a really slow/new relay. Try to measure it properly by
+        # using only relays with or above our calculated min_relay_bw (see:
+        # _calculate_min_bw_second_hop() in relaylist.py).
+        if min_bw < min_relay_bw:
+            min_bw = min_relay_bw
         new_candidates = stem_utils.only_relays_with_bandwidth(
             cont, candidates, min_bw=min_bw)
         if len(new_candidates) > 0:

sbws/lib/relaylist.py

@@ -278,6 +278,9 @@ class RelayList:
             [], MAX_RECENT_PRIORITY_RELAY_COUNT, state,
             "recent_measurement_attempt"
         )
+        # Start with 0 for the min bw for our second hops
+        self._exit_min_bw = 0
+        self._non_exit_min_bw = 0
         self._refresh()
 
     def _need_refresh(self):
@@ -431,6 +434,10 @@ class RelayList:
                  int(self._measurements_period / 24 / 60 / 60),
                  self.recent_consensus_count)
 
+        # Calculate minimum bandwidth value for 2nd hop after we refreshed
+        # our available relays.
+        self._calculate_min_bw_second_hop()
+
     @property
     def recent_consensus_count(self):
         """Number of times a new consensus was obtained."""
@@ -455,3 +462,31 @@ class RelayList:
     @property
     def recent_measurement_attempt_count(self):
         return len(self._recent_measurement_attempt)
+
+    def _calculate_min_bw_second_hop(self):
+        """
+        Calculates the minimum bandwidth for both exit and non-exit relays
+        chosen as a second hop by picking the lowest bandwidth value available
+        from the top 75% of the respective category.
+        """
+        # Sort our sets of candidates according to bw, lowest amount first.
+        # It's okay to keep things simple for the calculation and go over all
+        # exits, including badexits.
+        exit_candidates = sorted(self.exits,
+                                 key=lambda r: r.consensus_bandwidth)
+        non_exit_candidates = sorted(self.non_exits,
+                                     key=lambda r: r.consensus_bandwidth)
+        # We know the bandwidth is sorted from least to most. Dividing the
+        # length of the available relays by 4 gives us the position of the
+        # relay with the lowest bandwidth from the top 75%. We do this both
+        # for our exit and non-exit candidates.
+        pos = int(len(exit_candidates)/4)
+        self._exit_min_bw = exit_candidates[pos].consensus_bandwidth
+        pos = int(len(non_exit_candidates)/4)
+        self._non_exit_min_bw = non_exit_candidates[pos].consensus_bandwidth
+
+    def exit_min_bw(self):
+        return self._exit_min_bw
+
+    def non_exit_min_bw(self):
+        return self._non_exit_min_bw

tests/conftest.py

@@ -3,6 +3,7 @@ import pytest
 import os.path
 from unittest import mock
 
+from freezegun import freeze_time
 from stem import descriptor
 
 from sbws import settings
@@ -120,7 +121,8 @@ def router_status(server_descriptor, router_statuses):
 @pytest.fixture(scope='function')
 def relay_list(args, conf, controller):
     """Returns a RelayList containing the Relays in the controller"""
-    return relaylist.RelayList(args, conf, controller)
+    with freeze_time("2020-02-29 10:00:00"):
+        return relaylist.RelayList(args, conf, controller)
 
 
 @pytest.fixture(scope='function')

tests/unit/lib/test_relaylist.py

@@ -18,6 +18,8 @@ def test_init_relays(
     Test `init_relays` when creating the RelayList the first time and when a
     new consensus is received.
     Test that the number of consesus timesamps and relays is correct.
+    Additionally, make sure the calculated min bw for the second hop for
+    exit/non-exit relays is correct, too.
     """
     state = State(conf['paths']['state_fpath'])
     # There is no need to mock datetime to update the consensus, since the
@@ -31,6 +33,9 @@ def test_init_relays(
     # The actual number of relays in the consensus
     assert len(relay_list._relays) == 6433
     fps = {r.fingerprint for r in relay_list._relays}
+    # The calculated min bw for the second hop
+    assert 2100000 == relay_list._exit_min_bw
+    assert 220000 == relay_list._non_exit_min_bw
 
     # One hour later there is a new consensus
     relay_list._controller = controller_1h_later
@@ -44,6 +49,9 @@ def test_init_relays(
     fps_1h_later = {r.fingerprint for r in relay_list._relays}
     added_fps = fps_1h_later.difference(fps)
     assert 6505 == 6433 + len(added_fps)
+    # The calculated min bw for the second hop
+    assert 2120000 == relay_list._exit_min_bw
+    assert 200000 == relay_list._non_exit_min_bw
 
     # Five days later plus 1 second.
     # The first consensus timestamp will get removed.
@@ -62,6 +70,9 @@ def test_init_relays(
     # The number of relays will be the number of relays in the cosensus plus
     # the added ones minus the removed ones.
     assert 6925 == 6505 + len(added_fps) - len(removed_fps)
+    # The calculated min bw for the second hop
+    assert 2790000 == relay_list._exit_min_bw
+    assert 110000 == relay_list._non_exit_min_bw
 
 
 def test_increment_recent_measurement_attempt(args, conf, controller):
@@ -71,9 +82,8 @@ def test_increment_recent_measurement_attempt(args, conf, controller):
     It also tests that the state file is updated correctly.
     """
     state = State(conf['paths']['state_fpath'])
-    # For this test it does not matter that the consensus timestamps or relays
-    # are not correct.
-    relay_list = RelayList(args, conf, controller=controller, state=state)
+    with freeze_time("2020-02-29 10:00:00"):
+        relay_list = RelayList(args, conf, controller=controller, state=state)
     # The initial count is 0 and the state does not have that key.
     assert 0 == relay_list.recent_measurement_attempt_count
     assert not state.get("recent_measurement_attempt", None)