GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Commit e098ebb9 authored by juga  's avatar juga 💬

chg: relaylist, v3bwfile: Count consensus with timestamps

in RelayList:
- Rename consensus_timestamps to recent_consensus
- Rename recent_consensus_count to recent_consensus when there is
  no counting
- Use the timestamps class to manage/count consensuses
- Remove method not needed anymore
parent 65589118
......@@ -148,6 +148,10 @@ MAX_NUM_DESTINATION_FAILURES = 3
# destination fail again.
FACTOR_INCREMENT_DESTINATION_RETRY = 2
# Constants to check health KeyValues in the bandwidth file
PERIOD_DAYS = int(MEASUREMENTS_PERIOD / (24 * 60 * 60))
MAX_RECENT_CONSENSUS_COUNT = PERIOD_DAYS * 24 # 120
def fail_hard(*a, **kw):
''' Log something ... and then exit as fast as possible '''
......
......@@ -8,8 +8,11 @@ import random
import logging
from threading import Lock
from ..globals import MEASUREMENTS_PERIOD
from ..util import timestamp
from ..globals import (
MAX_RECENT_CONSENSUS_COUNT,
MEASUREMENTS_PERIOD
)
from ..util import timestamp, timestamps
log = logging.getLogger(__name__)
......@@ -322,7 +325,9 @@ class RelayList:
self.rng = random.SystemRandom()
self._refresh_lock = Lock()
# To track all the consensus seen.
self._consensus_timestamps = []
self._recent_consensus = timestamps.DateTimeSeq(
[], MAX_RECENT_CONSENSUS_COUNT, state, "recent_consensus"
)
# Initialize so that there's no error trying to access to it.
# In future refactor, change to a dictionary, where the keys are
# the relays' fingerprint.
......@@ -345,15 +350,7 @@ class RelayList:
@property
def last_consensus_timestamp(self):
"""Returns the datetime when the last consensus was obtained."""
if (getattr(self, "_consensus_timestamps")
and self._consensus_timestamps):
return self._consensus_timestamps[-1]
# If the object was not created from __init__, it won't have
# consensus_timestamps attribute or it might be empty.
# In this case force new update.
# Anytime more than 1h in the past will be old.
self._consensus_timestamps = []
return datetime.utcnow() - timedelta(seconds=60*61)
return self._recent_consensus.last()
@property
def relays(self):
......@@ -419,12 +416,6 @@ class RelayList:
def _relays_without_flag(self, flag):
return [r for r in self.relays if flag not in r.flags]
def _remove_old_consensus_timestamps(self):
self._consensus_timestamps = remove_old_consensus_timestamps(
copy.deepcopy(self._consensus_timestamps),
self._measurements_period
)
def _init_relays(self):
"""Returns a new list of relays that are in the current consensus.
And update the consensus timestamp list with the current one.
......@@ -441,8 +432,7 @@ class RelayList:
# Find the timestamp of the last consensus.
timestamp = valid_after_from_network_statuses(network_statuses)
self._consensus_timestamps.append(timestamp)
self._remove_old_consensus_timestamps()
self._recent_consensus.update(timestamp)
new_relays = []
......@@ -502,14 +492,11 @@ class RelayList:
log.info("Number of consensuses obtained in the last %s days: %s.",
int(self._measurements_period / 24 / 60 / 60),
self.recent_consensus_count)
# NOTE: blocking, writes to file!
if self._state is not None:
self._state['recent_consensus_count'] = self.recent_consensus_count
@property
def recent_consensus_count(self):
"""Number of times a new consensus was obtained."""
return len(self._consensus_timestamps)
return len(self._recent_consensus)
def exits_not_bad_allowing_port(self, port):
return [r for r in self.exits
......
......@@ -367,7 +367,7 @@ class V3BWHeader(object):
if destinations_countries is not None:
kwargs['destinations_countries'] = destinations_countries
if recent_consensus_count is not None:
kwargs['recent_consensus_count'] = str(recent_consensus_count)
kwargs['recent_consensus_count'] = recent_consensus_count
recent_measurement_attempt_count = \
cls.recent_measurement_attempt_count_from_file(state_fpath)
......@@ -457,10 +457,10 @@ class V3BWHeader(object):
@staticmethod
def consensus_count_from_file(state_fpath):
state = State(state_fpath)
if 'recent_consensus_count' in state:
return state['recent_consensus_count']
else:
return None
count = state.count("recent_consensus")
if count:
return str(count)
return None
# NOTE: in future refactor store state in the class
@staticmethod
......
{
"uuid": "806218a0-3ce5-4778-b839-d6faf6798405",
"scanner_started": "2019-03-25T13:03:06",
"recent_consensus_count": 1,
"recent_priority_list_count": 1,
"recent_measurement_attempt_count": 15,
"min_perc_reached": null,
"recent_priority_relay_count": 15
}
\ No newline at end of file
"scanner_started": "2020-02-29T10:00:00",
"uuid": "e4ecf294-f253-478c-8660-28cbdfc690de",
"tor_version": "0.4.2.6",
"recent_consensus": [
"2020-03-18T13:26:46"
],
"recent_priority_list": [
"2020-03-18T13:26:46"
],
"recent_priority_relay": [
[
"2020-03-18T13:26:46",
15
]
],
"recent_measurement_attempt": [
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46",
"2020-03-18T13:26:46"
],
"min_perc_reached": null
}
......@@ -8,6 +8,7 @@ from datetime import datetime, timedelta
from freezegun import freeze_time
from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps
from sbws.util.state import State
def test_remove_old_consensus_timestamps():
......@@ -31,13 +32,14 @@ def test_init_relays(
new consensus is received.
Test that the number of consesus timesamps and relays is correct.
"""
state = State(conf['paths']['state_fpath'])
# There is no need to mock datetime to update the consensus, since the
# actual date will be always later.
# But it's needed to have the correct list of timestamps both for RelayList
# and Relay.
with freeze_time("2020-02-29 10:00:00"):
relay_list = RelayList(args, conf, controller=controller)
assert len(relay_list._consensus_timestamps) == 1
relay_list = RelayList(args, conf, controller, state=state)
assert relay_list.recent_consensus_count == 1
assert len(relay_list._relays[0]._consensus_timestamps) == 1
# The actual number of relays in the consensus
assert len(relay_list._relays) == 6433
......@@ -48,7 +50,7 @@ def test_init_relays(
with freeze_time("2020-02-29 11:00:00"):
# Call relays update the list of relays.
relay_list.relays
assert len(relay_list._consensus_timestamps) == 2
assert relay_list.recent_consensus_count == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2
# Check that the number of relays is now the previous one plus the relays
# that are in the new consensus that there were not in the previous one.
......@@ -61,7 +63,7 @@ def test_init_relays(
relay_list._controller = controller_5days_later
with freeze_time("2020-03-05 10:00:01"):
relay_list.relays
assert len(relay_list._consensus_timestamps) == 2
assert relay_list.recent_consensus_count == 2
assert len(relay_list._relays[0]._consensus_timestamps) == 2
fps_5days_later = {r.fingerprint for r in relay_list._relays}
# The number of added relays will be the number of relays in this
......
......@@ -16,6 +16,7 @@ from sbws.lib.v3bwfile import (
V3BWFile, round_sig_dig,
HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS
)
from sbws.util.state import CustomDecoder
from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts
timestamp = 1523974147
......@@ -248,7 +249,7 @@ def test_v3bwline_from_results_file(datadir):
lines = datadir.readlines('results.txt')
d = dict()
for line in lines:
r = Result.from_dict(json.loads(line.strip()))
r = Result.from_dict(json.loads(line.strip(), cls=CustomDecoder))
fp = r.fingerprint
if fp not in d:
d[fp] = []
......@@ -521,11 +522,21 @@ def test_set_under_min_report(mock_consensus, conf, datadir):
def test_generator_started(root_data_path, datadir):
state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
# The method is correct
assert "2019-03-25T13:03:06" == V3BWHeader.generator_started_from_file(
assert "2020-02-29T10:00:00" == V3BWHeader.generator_started_from_file(
state_fpath
)
# `results` does not matter here, using them to not have an empty list.
results = load_result_file(str(datadir.join("results.txt")))
header = V3BWHeader.from_results(results, '', '', state_fpath)
# And the header is correct
assert "2019-03-25T13:03:06" == header.generator_started
assert "2020-02-29T10:00:00" == header.generator_started
def test_recent_consensus_count(root_data_path, datadir):
# This state has recent_consensus_count
state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
assert "1" == V3BWHeader.consensus_count_from_file(state_fpath)
# `results` does not matter here, using them to not have an empty list.
results = load_result_file(str(datadir.join("results.txt")))
header = V3BWHeader.from_results(results, '', '', state_fpath)
assert "1" == header.recent_consensus_count
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment