GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still and

Commit 205eefd1 authored by juga  's avatar juga
Browse files

Merge branch 'maint-1.1'

parents 44ece919 c36518cb
......@@ -138,6 +138,9 @@ NUM_DESTINATION_ATTEMPTS_STORED = 10
# Because intermitent failures with CDN destinations, start trying again
# after 5 min.
# No matter what, do not increase the wait time between destination reties
# past this value.
# Number of consecutive times a destination can fail before considering it
# not functional.
......@@ -11,6 +11,7 @@ import sbws.util.stem as stem_utils
from ..globals import (
......@@ -121,7 +122,7 @@ def connect_to_destination_over_circuit(dest, circ_id, session, cont, max_dl):
'{} not {}'.format(, head.status_code)
if 'content-length' not in head.headers:
return False, error_prefix + 'we except the header Content-Length '\
return False, error_prefix + 'we expect the header Content-Length '\
'to exist in the response'
content_length = int(head.headers['content-length'])
if max_dl > content_length:
......@@ -143,6 +144,7 @@ class Destination:
def __init__(self, url, max_dl, verify,
"""Initalizes the Web server from which the data is downloaded.
......@@ -169,6 +171,8 @@ class Destination:
# Default delta time to try a destination that was not functional.
self._default_delta_seconds_retry = delta_seconds_retry
self._delta_seconds_retry = delta_seconds_retry
# A cap on the time to wait between destination retries.
self._max_seconds_between_retries = max_seconds_between_retries
# Using a deque (FIFO) to do not grow forever and
# to do not have to remove old attempts.
# Store tuples of timestamp and whether the destination succed or not
......@@ -201,20 +205,30 @@ class Destination:
Increment the time a destination will be tried again by a ``factor``.
self._delta_seconds_retry *= factor or self._factor"Incremented the time to try destination %s to %s hours.",
self.url, self._delta_seconds_retry / 60 / 60)
if self._delta_seconds_retry > self._max_seconds_between_retries:
self._delta_seconds_retry = self._max_seconds_between_retries"Incremented the time to try destination %s past the "
"limit, capping it at %s hours.",
self.url, self._delta_seconds_retry / 60 / 60)
else:"Incremented the time to try destination %s to %s hours.",
self.url, self._delta_seconds_retry / 60 / 60)
def _is_last_try_old_enough(self, n=None):
def _get_last_try_in_seconds_ago(self):
Return True if the last time it was used it was ``n`` seconds ago.
Return the delta between the last try and now, as positive seconds.
# Timestamp of the last attempt.
last_time = self._attempts[-1][0]
return (datetime.datetime.utcnow() - last_time).total_seconds()
def _is_last_try_old_enough(self, n=None):
Return True if the last time it was used it was ``n`` seconds ago.
# If the last attempt is older than _delta_seconds_retry, try again
return (datetime.datetime.utcnow()
- datetime.timedelta(seconds=self._delta_seconds_retry)
> last_time)
return False
return (self._get_last_try_in_seconds_ago() >
def is_functional(self):
"""Whether connections to a destination are failing or not.
......@@ -238,18 +252,21 @@ class Destination:
if self._are_last_attempts_failures():
# The log here will appear in all the the queued
# relays and threads.
log.warning("The last %s times the destination %s failed."
"Disabled for %s minutes.",
log.warning("The last %s times the destination %s failed. "
"It last ran %s seconds ago. "
"Disabled for %s seconds.",
self._max_num_failures, self.url,
self._delta_seconds_retry / 60)
log.warning("Please, add more destinations or increment the "
"number of maximum number of consecutive failures "
"in the configuration.")
# It was not used for a while and the last time it was used
# was long ago, then try again
if self._is_last_try_old_enough():"The destination %s was not tried for %s hours, "
"it is going to by tried again.")"The destination %s was not tried for %s seconds, "
"it is going to by tried again.", self.url,
# Set the next time to retry higher, in case this attempt fails
return True
"""Unit tests for sbws.lib.destination."""
from datetime import datetime, timedelta
from sbws.globals import MAX_SECONDS_RETRY_DESTINATION
from sbws.lib import destination
......@@ -8,6 +9,12 @@ def test_destination_is_functional():
eleven_mins_ago = datetime.utcnow() - timedelta(minutes=11)
six_mins_ago = datetime.utcnow() - timedelta(minutes=6)
four_mins_ago = datetime.utcnow() - timedelta(minutes=4)
# Make last time tried a bit bigger than the half of the maximum, so that
# it's bigger than the delta time to retry, and when delta time to retry
# is muliplied by a factor (2) it reaches the maximum.
long_ago = datetime.utcnow() - timedelta(
d = destination.Destination('unexistenturl', 0, False)
assert d.is_functional()
......@@ -52,3 +59,16 @@ def test_destination_is_functional():
assert d.is_functional()
# And the delta to try is resetted
assert not d._is_last_try_old_enough()
# When the delta time to retry a destination increase too much,
# set it to a maximum, and try the destination again
# Pretend the delta seconds was already set to a bit more than
# half the maximum.
d._delta_seconds_retry = (MAX_SECONDS_RETRY_DESTINATION / 2) + 1
assert d._are_last_attempts_failures()
assert d._is_last_try_old_enough()
assert d.is_functional()
assert d._delta_seconds_retry == MAX_SECONDS_RETRY_DESTINATION
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment