chg: relaylist: Count measurements with timestamps

in RelayList:
- Rename recent_measurement_attempt_count to recent_measurement_attempt when
  there is no counting
- Use the timestamps class to manage/count measurement attempts

sbws/core/scanner.py

@@ -515,7 +515,7 @@ def main_loop(args, conf, controller, relay_list, circuit_builder, result_dump,
             # Don't start measuring a relay if sbws is stopping.
             if settings.end_event.is_set():
                 break
-            relay_list.increment_recent_measurement_attempt_count()
+            relay_list.increment_recent_measurement_attempt()
             target.increment_relay_recent_measurement_attempt_count()
             num_relays += 1
             # callback and callback_err must be non-blocking

sbws/globals.py

@@ -151,6 +151,22 @@ FACTOR_INCREMENT_DESTINATION_RETRY = 2
 # Constants to check health KeyValues in the bandwidth file
 PERIOD_DAYS = int(MEASUREMENTS_PERIOD / (24 * 60 * 60))
 MAX_RECENT_CONSENSUS_COUNT = PERIOD_DAYS * 24  # 120
+# XXX: This was only defined in `config.default.ini`, it should be read from
+# here.
+FRACTION_RELAYS = 0.05
+# A priority list currently takes more than 3h, ideally it should only take 1h.
+MIN_HOURS_PRIORITY_LIST = 1
+# As of 2020, there're less than 7000 relays.
+MAX_RELAYS = 8000
+# 120
+MAX_RECENT_PRIORITY_LIST_COUNT = int(
+    PERIOD_DAYS * 24 / MIN_HOURS_PRIORITY_LIST
+)
+MAX_RELAYS_PER_PRIORITY_LIST = int(MAX_RELAYS * FRACTION_RELAYS)  # 400
+# 48000
+MAX_RECENT_PRIORITY_RELAY_COUNT = (
+    MAX_RECENT_PRIORITY_LIST_COUNT * MAX_RELAYS_PER_PRIORITY_LIST
+)
 
 
 def fail_hard(*a, **kw):

sbws/lib/relaylist.py

@@ -10,6 +10,7 @@ from threading import Lock
 
 from ..globals import (
     MAX_RECENT_CONSENSUS_COUNT,
+    MAX_RECENT_PRIORITY_RELAY_COUNT,
     MEASUREMENTS_PERIOD
 )
 from ..util import timestamp, timestamps
@@ -334,12 +335,10 @@ class RelayList:
         self._relays = []
         # The period of time for which the measurements are keep.
         self._measurements_period = measurements_period
-        self._state = state
-        # NOTE: blocking: writes to disk
-        if self._state:
-            if self._state.get('recent_measurement_attempt_count', None) \
-                    is None:
-                self._state['recent_measurement_attempt_count'] = 0
+        self._recent_measurement_attempt = timestamps.DateTimeSeq(
+            [], MAX_RECENT_PRIORITY_RELAY_COUNT, state,
+            "recent_measurement_attempt"
+        )
         self._refresh()
 
     def _need_refresh(self):
@@ -502,7 +501,7 @@ class RelayList:
         return [r for r in self.exits
                 if r.is_exit_not_bad_allowing_port(port)]
 
-    def increment_recent_measurement_attempt_count(self):
+    def increment_recent_measurement_attempt(self):
         """
         Increment the number of times that any relay has been queued to be
         measured.
@@ -512,5 +511,8 @@ class RelayList:
         It is read and stored in a ``state`` file.
         """
         # NOTE: blocking, writes to file!
-        if self._state:
-            self._state['recent_measurement_attempt_count'] += 1
+        self._recent_measurement_attempt.update()
+
+    @property
+    def recent_measurement_attempt_count(self):
+        return len(self._recent_measurement_attempt)

sbws/lib/v3bwfile.py

@@ -470,7 +470,7 @@ class V3BWHeader(object):
         in the recent (by default 5) days from the state file.
         """
         state = State(state_fpath)
-        return state.get('recent_measurement_attempt_count', None)
+        return state.count('recent_measurement_attempt')
 
     @staticmethod
     def recent_priority_list_count_from_file(state_fpath):

tests/unit/lib/test_relaylist.py

@@ -75,3 +75,45 @@ def test_init_relays(
     # The number of relays will be the number of relays in the cosensus plus
     # the added ones minus the removed ones.
     assert 6925 == 6505 + len(added_fps) - len(removed_fps)
+
+
+def test_increment_recent_measurement_attempt(args, conf, controller):
+    """Test that incrementing the measurement attempts does not go on forever
+
+    And instead it only counts the number of attempts in the last days.
+    It also tests that the state file is updated correctly.
+    """
+    state = State(conf['paths']['state_fpath'])
+    # For this test it does not matter that the consensus timestamps or relays
+    # are not correct.
+    relay_list = RelayList(args, conf, controller=controller, state=state)
+    # The initial count is 0 and the state does not have that key.
+    assert 0 == relay_list.recent_measurement_attempt_count
+    assert not state.get("recent_measurement_attempt", None)
+
+    # Pretend that a measurement attempt is made.
+    with freeze_time("2020-02-29 10:00:00"):
+        relay_list.increment_recent_measurement_attempt()
+    assert 1 == relay_list.recent_measurement_attempt_count
+    assert [datetime(2020, 2, 29, 10, 0)] == state[
+        "recent_measurement_attempt"
+    ]
+
+    # And a second measurement attempt is made 4 days later.
+    with freeze_time("2020-03-04 10:00:00"):
+        relay_list.increment_recent_measurement_attempt()
+    assert 2 == relay_list.recent_measurement_attempt_count
+    assert 2 == len(state["recent_measurement_attempt"])
+
+    # And a third measurement attempt is made 5 days later.
+    with freeze_time("2020-03-05 10:00:00"):
+        relay_list.increment_recent_measurement_attempt()
+    assert 3 == relay_list.recent_measurement_attempt_count
+    assert 3 == len(state["recent_measurement_attempt"])
+
+    # And a fourth measurement attempt is made 6 days later. The first one is
+    # now removed and not counted.
+    with freeze_time("2020-03-06 10:00:00"):
+        relay_list.increment_recent_measurement_attempt()
+    assert 3 == relay_list.recent_measurement_attempt_count
+    assert 3 == len(state["recent_measurement_attempt"])

tests/unit/lib/test_v3bwfile.py

@@ -540,3 +540,14 @@ def test_recent_consensus_count(root_data_path, datadir):
     results = load_result_file(str(datadir.join("results.txt")))
     header = V3BWHeader.from_results(results, '', '', state_fpath)
     assert "1" == header.recent_consensus_count
+
+
+def test_recent_measurement_attempt_count(root_data_path, datadir):
+    state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
+    assert 15 == V3BWHeader.recent_measurement_attempt_count_from_file(
+        state_fpath
+    )
+    # `results` does not matter here, using them to not have an empty list.
+    results = load_result_file(str(datadir.join("results.txt")))
+    header = V3BWHeader.from_results(results, '', '', state_fpath)
+    assert "15" == header.recent_measurement_attempt_count