Commit a550d731 authored by juga's avatar juga
Browse files

scanner: log backtrace when not progressing

When sbws stalls after the prioritization loop waiting for thread
results, give it some minutes and otherwise log the backtrace.
This way if there's any bug or deadlock, operators can help to
debug it.
parent ee64d76d
''' Measure the relays. '''
import sys
import threading
from ..lib.circuitbuilder import GapsCircuitBuilder as CB
from ..lib.resultdump import ResultDump
from ..lib.resultdump import ResultSuccess, ResultErrorCircuit
......@@ -27,6 +30,20 @@ end_event = Event()
log = logging.getLogger(__name__)
def dumpstacks():
import pdb
import traceback
log.warning("sbws stop measuring relays, probably because of a bug."
"Please, open a ticket in trac.torproject.org with this"
"backtrace.")
thread_id2name = dict([(t.ident, t.name) for t in threading.enumerate()])
for thread_id, stack in sys._current_frames().items():
log.critical("Thread: %s(%d)",
thread_id2name.get(thread_id, ""), thread_id)
log.critical(traceback.print_stack(stack))
pdb.set_trace()
def timed_recv_from_server(session, dest, byte_range):
''' Request the **byte_range** from the URL at **dest**. If successful,
return True and the time it took to download. Otherwise return False and an
......@@ -359,9 +376,16 @@ def run_speedtest(args, conf):
while len(pending_results) >= max_pending_results:
time.sleep(5)
pending_results = [r for r in pending_results if not r.ready()]
while len(pending_results) > 0:
counter = 0
# give it 3min, otherwise there's a bug or deadlock
while len(pending_results) > 0 and counter <= 36:
counter += 1
log.debug("Number of pending measurement threads %s after "
"a prioritization loop.", len(pending_results))
time.sleep(5)
pending_results = [r for r in pending_results if not r.ready()]
if counter > 36:
dumpstacks()
loop_tstop = time.time()
loop_tdelta = (loop_tstop - loop_tstart) / 60
log.debug("Measured %s relays in %s minutes", num_relays, loop_tdelta)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment