GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Commit e098ebb9 authored by juga  's avatar juga

chg: relaylist, v3bwfile: Count consensus with timestamps

in RelayList:
- Rename consensus_timestamps to recent_consensus
- Rename recent_consensus_count to recent_consensus when there is
  no counting
- Use the timestamps class to manage/count consensuses
- Remove method not needed anymore
parent 65589118
...@@ -148,6 +148,10 @@ MAX_NUM_DESTINATION_FAILURES = 3 ...@@ -148,6 +148,10 @@ MAX_NUM_DESTINATION_FAILURES = 3
# destination fail again. # destination fail again.
FACTOR_INCREMENT_DESTINATION_RETRY = 2 FACTOR_INCREMENT_DESTINATION_RETRY = 2
# Constants to check health KeyValues in the bandwidth file
PERIOD_DAYS = int(MEASUREMENTS_PERIOD / (24 * 60 * 60))
MAX_RECENT_CONSENSUS_COUNT = PERIOD_DAYS * 24 # 120
def fail_hard(*a, **kw): def fail_hard(*a, **kw):
''' Log something ... and then exit as fast as possible ''' ''' Log something ... and then exit as fast as possible '''
......
...@@ -8,8 +8,11 @@ import random ...@@ -8,8 +8,11 @@ import random
import logging import logging
from threading import Lock from threading import Lock
from ..globals import MEASUREMENTS_PERIOD from ..globals import (
from ..util import timestamp MAX_RECENT_CONSENSUS_COUNT,
MEASUREMENTS_PERIOD
)
from ..util import timestamp, timestamps
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
...@@ -322,7 +325,9 @@ class RelayList: ...@@ -322,7 +325,9 @@ class RelayList:
self.rng = random.SystemRandom() self.rng = random.SystemRandom()
self._refresh_lock = Lock() self._refresh_lock = Lock()
# To track all the consensus seen. # To track all the consensus seen.
self._consensus_timestamps = [] self._recent_consensus = timestamps.DateTimeSeq(
[], MAX_RECENT_CONSENSUS_COUNT, state, "recent_consensus"
)
# Initialize so that there's no error trying to access to it. # Initialize so that there's no error trying to access to it.
# In future refactor, change to a dictionary, where the keys are # In future refactor, change to a dictionary, where the keys are
# the relays' fingerprint. # the relays' fingerprint.
...@@ -345,15 +350,7 @@ class RelayList: ...@@ -345,15 +350,7 @@ class RelayList:
@property @property
def last_consensus_timestamp(self): def last_consensus_timestamp(self):
"""Returns the datetime when the last consensus was obtained.""" """Returns the datetime when the last consensus was obtained."""
if (getattr(self, "_consensus_timestamps") return self._recent_consensus.last()
and self._consensus_timestamps):
return self._consensus_timestamps[-1]
# If the object was not created from __init__, it won't have
# consensus_timestamps attribute or it might be empty.
# In this case force new update.
# Anytime more than 1h in the past will be old.
self._consensus_timestamps = []
return datetime.utcnow() - timedelta(seconds=60*61)
@property @property
def relays(self): def relays(self):
...@@ -419,12 +416,6 @@ class RelayList: ...@@ -419,12 +416,6 @@ class RelayList:
def _relays_without_flag(self, flag): def _relays_without_flag(self, flag):
return [r for r in self.relays if flag not in r.flags] return [r for r in self.relays if flag not in r.flags]
def _remove_old_consensus_timestamps(self):
self._consensus_timestamps = remove_old_consensus_timestamps(
copy.deepcopy(self._consensus_timestamps),
self._measurements_period
)
def _init_relays(self): def _init_relays(self):
"""Returns a new list of relays that are in the current consensus. """Returns a new list of relays that are in the current consensus.
And update the consensus timestamp list with the current one. And update the consensus timestamp list with the current one.
...@@ -441,8 +432,7 @@ class RelayList: ...@@ -441,8 +432,7 @@ class RelayList:
# Find the timestamp of the last consensus. # Find the timestamp of the last consensus.
timestamp = valid_after_from_network_statuses(network_statuses) timestamp = valid_after_from_network_statuses(network_statuses)
self._consensus_timestamps.append(timestamp) self._recent_consensus.update(timestamp)
self._remove_old_consensus_timestamps()
new_relays = [] new_relays = []
...@@ -502,14 +492,11 @@ class RelayList: ...@@ -502,14 +492,11 @@ class RelayList:
log.info("Number of consensuses obtained in the last %s days: %s.", log.info("Number of consensuses obtained in the last %s days: %s.",
int(self._measurements_period / 24 / 60 / 60), int(self._measurements_period / 24 / 60 / 60),
self.recent_consensus_count) self.recent_consensus_count)
# NOTE: blocking, writes to file!
if self._state is not None:
self._state['recent_consensus_count'] = self.recent_consensus_count
@property @property
def recent_consensus_count(self): def recent_consensus_count(self):
"""Number of times a new consensus was obtained.""" """Number of times a new consensus was obtained."""
return len(self._consensus_timestamps) return len(self._recent_consensus)
def exits_not_bad_allowing_port(self, port): def exits_not_bad_allowing_port(self, port):
return [r for r in self.exits return [r for r in self.exits
......
...@@ -367,7 +367,7 @@ class V3BWHeader(object): ...@@ -367,7 +367,7 @@ class V3BWHeader(object):
if destinations_countries is not None: if destinations_countries is not None:
kwargs['destinations_countries'] = destinations_countries kwargs['destinations_countries'] = destinations_countries
if recent_consensus_count is not None: if recent_consensus_count is not None:
kwargs['recent_consensus_count'] = str(recent_consensus_count) kwargs['recent_consensus_count'] = recent_consensus_count
recent_measurement_attempt_count = \ recent_measurement_attempt_count = \
cls.recent_measurement_attempt_count_from_file(state_fpath) cls.recent_measurement_attempt_count_from_file(state_fpath)
...@@ -457,10 +457,10 @@ class V3BWHeader(object): ...@@ -457,10 +457,10 @@ class V3BWHeader(object):
@staticmethod @staticmethod
def consensus_count_from_file(state_fpath): def consensus_count_from_file(state_fpath):
state = State(state_fpath) state = State(state_fpath)
if 'recent_consensus_count' in state: count = state.count("recent_consensus")
return state['recent_consensus_count'] if count:
else: return str(count)
return None return None
# NOTE: in future refactor store state in the class # NOTE: in future refactor store state in the class
@staticmethod @staticmethod
......
{ {
"uuid": "806218a0-3ce5-4778-b839-d6faf6798405", "scanner_started": "2020-02-29T10:00:00",
"scanner_started": "2019-03-25T13:03:06", "uuid": "e4ecf294-f253-478c-8660-28cbdfc690de",
"recent_consensus_count": 1, "tor_version": "0.4.2.6",
"recent_priority_list_count": 1, "recent_consensus": [
"recent_measurement_attempt_count": 15, "2020-03-18T13:26:46"
"min_perc_reached": null, ],
"recent_priority_relay_count": 15 "recent_priority_list": [
} "2020-03-18T13:26:46"
\ No newline at end of file ],
"recent_priority_relay": [
[
"2020-03-18T13:26:46",
15
]
],
"recent_measurement_attempt": [
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46"
],
"min_perc_reached": null
}
...@@ -8,6 +8,7 @@ from datetime import datetime, timedelta ...@@ -8,6 +8,7 @@ from datetime import datetime, timedelta
from freezegun import freeze_time from freezegun import freeze_time
from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps
from sbws.util.state import State
def test_remove_old_consensus_timestamps(): def test_remove_old_consensus_timestamps():
...@@ -31,13 +32,14 @@ def test_init_relays( ...@@ -31,13 +32,14 @@ def test_init_relays(
new consensus is received. new consensus is received.
Test that the number of consesus timesamps and relays is correct. Test that the number of consesus timesamps and relays is correct.
""" """
state = State(conf['paths']['state_fpath'])
# There is no need to mock datetime to update the consensus, since the # There is no need to mock datetime to update the consensus, since the
# actual date will be always later. # actual date will be always later.
# But it's needed to have the correct list of timestamps both for RelayList # But it's needed to have the correct list of timestamps both for RelayList
# and Relay. # and Relay.
with freeze_time("2020-02-29 10:00:00"): with freeze_time("2020-02-29 10:00:00"):
relay_list = RelayList(args, conf, controller=controller) relay_list = RelayList(args, conf, controller, state=state)
assert len(relay_list._consensus_timestamps) == 1 assert relay_list.recent_consensus_count == 1
assert len(relay_list._relays[0]._consensus_timestamps) == 1 assert len(relay_list._relays[0]._consensus_timestamps) == 1
# The actual number of relays in the consensus # The actual number of relays in the consensus
assert len(relay_list._relays) == 6433 assert len(relay_list._relays) == 6433
...@@ -48,7 +50,7 @@ def test_init_relays( ...@@ -48,7 +50,7 @@ def test_init_relays(
with freeze_time("2020-02-29 11:00:00"): with freeze_time("2020-02-29 11:00:00"):
# Call relays update the list of relays. # Call relays update the list of relays.
relay_list.relays relay_list.relays
assert len(relay_list._consensus_timestamps) == 2 assert relay_list.recent_consensus_count == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2 assert len(relay_list._relays[0]._consensus_timestamps) == 2
# Check that the number of relays is now the previous one plus the relays # Check that the number of relays is now the previous one plus the relays
# that are in the new consensus that there were not in the previous one. # that are in the new consensus that there were not in the previous one.
...@@ -61,7 +63,7 @@ def test_init_relays( ...@@ -61,7 +63,7 @@ def test_init_relays(
relay_list._controller = controller_5days_later relay_list._controller = controller_5days_later
with freeze_time("2020-03-05 10:00:01"): with freeze_time("2020-03-05 10:00:01"):
relay_list.relays relay_list.relays
assert len(relay_list._consensus_timestamps) == 2 assert relay_list.recent_consensus_count == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2 assert len(relay_list._relays[0]._consensus_timestamps) == 2
fps_5days_later = {r.fingerprint for r in relay_list._relays} fps_5days_later = {r.fingerprint for r in relay_list._relays}
# The number of added relays will be the number of relays in this # The number of added relays will be the number of relays in this
......
...@@ -16,6 +16,7 @@ from sbws.lib.v3bwfile import ( ...@@ -16,6 +16,7 @@ from sbws.lib.v3bwfile import (
V3BWFile, round_sig_dig, V3BWFile, round_sig_dig,
HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS
) )
from sbws.util.state import CustomDecoder
from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts
timestamp = 1523974147 timestamp = 1523974147
...@@ -248,7 +249,7 @@ def test_v3bwline_from_results_file(datadir): ...@@ -248,7 +249,7 @@ def test_v3bwline_from_results_file(datadir):
lines = datadir.readlines('results.txt') lines = datadir.readlines('results.txt')
d = dict() d = dict()
for line in lines: for line in lines:
r = Result.from_dict(json.loads(line.strip())) r = Result.from_dict(json.loads(line.strip(), cls=CustomDecoder))
fp = r.fingerprint fp = r.fingerprint
if fp not in d: if fp not in d:
d[fp] = [] d[fp] = []
...@@ -521,11 +522,21 @@ def test_set_under_min_report(mock_consensus, conf, datadir): ...@@ -521,11 +522,21 @@ def test_set_under_min_report(mock_consensus, conf, datadir):
def test_generator_started(root_data_path, datadir): def test_generator_started(root_data_path, datadir):
state_fpath = os.path.join(root_data_path, '.sbws/state.dat') state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
# The method is correct # The method is correct
assert "2019-03-25T13:03:06" == V3BWHeader.generator_started_from_file( assert "2020-02-29T10:00:00" == V3BWHeader.generator_started_from_file(
state_fpath state_fpath
) )
# `results` does not matter here, using them to not have an empty list. # `results` does not matter here, using them to not have an empty list.
results = load_result_file(str(datadir.join("results.txt"))) results = load_result_file(str(datadir.join("results.txt")))
header = V3BWHeader.from_results(results, '', '', state_fpath) header = V3BWHeader.from_results(results, '', '', state_fpath)
# And the header is correct # And the header is correct
assert "2019-03-25T13:03:06" == header.generator_started assert "2020-02-29T10:00:00" == header.generator_started
def test_recent_consensus_count(root_data_path, datadir):
# This state has recent_consensus_count
state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
assert "1" == V3BWHeader.consensus_count_from_file(state_fpath)
# `results` does not matter here, using them to not have an empty list.
results = load_result_file(str(datadir.join("results.txt")))
header = V3BWHeader.from_results(results, '', '', state_fpath)
assert "1" == header.recent_consensus_count
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment