Merge branch 'bug28741'

There was merge conflict in the imports, solved adding all of them

sbws/__init__.py

 __version__ = '1.0.3-dev0'
+
+from . import globals  # noqa
+
+
+class Settings:
+    def __init__(self):
+        # update this dict from globals (but only for ALL_CAPS settings)
+        for setting in dir(globals):
+            if setting.isupper():
+                setattr(self, setting, getattr(globals, setting))
+
+    def init_http_headers(self, nickname, uuid, tor_version):
+        self.HTTP_HEADERS['Tor-Bandwidth-Scanner-Nickname'] = nickname
+        self.HTTP_HEADERS['Tor-Bandwidth-Scanner-UUID'] = uuid
+        self.HTTP_HEADERS['User-Agent'] += tor_version
+
+settings = Settings()  # noqa

sbws/core/scanner.py

@@ -2,6 +2,7 @@
 
 import sys
 import threading
+import uuid
 
 from ..lib.circuitbuilder import GapsCircuitBuilder as CB
 from ..lib.resultdump import ResultDump
@@ -12,7 +13,7 @@ from ..lib.relayprioritizer import RelayPrioritizer
 from ..lib.destination import DestinationList
 from ..util.timestamp import now_isodt_str
 from ..util.state import State
-from sbws.globals import fail_hard, TIMEOUT_MEASUREMENTS
+from sbws.globals import fail_hard, TIMEOUT_MEASUREMENTS, HTTP_GET_HEADERS
 import sbws.util.stem as stem_utils
 import sbws.util.requests as requests_utils
 from argparse import ArgumentDefaultsHelpFormatter
@@ -24,6 +25,8 @@ import logging
 import requests
 import random
 
+from sbws import settings
+
 
 rng = random.SystemRandom()
 end_event = Event()
@@ -55,17 +58,21 @@ def timed_recv_from_server(session, dest, byte_range):
     ''' Request the **byte_range** from the URL at **dest**. If successful,
     return True and the time it took to download. Otherwise return False and an
     exception. '''
-    headers = {'Range': byte_range, 'Accept-Encoding': 'identity'}
+
     start_time = time.time()
+    HTTP_GET_HEADERS['Range'] = byte_range
     # TODO:
     # - What other exceptions can this throw?
-    # - Do we have to read the content, or did requests already do so?
+    # - response.elapsed "measures the time taken between sending the first
+    #   byte of the request and finishing parsing the headers.
+    #   It is therefore unaffected by consuming the response content"
+    #   If this mean that the content has arrived, elapsed could be used to
+    #   know the time it took.
     try:
-        requests_utils.get(
-            session, dest.url, headers=headers, verify=dest.verify)
-    except requests.exceptions.ConnectionError as e:
-        return False, e
-    except requests.exceptions.ReadTimeout as e:
+        # headers are merged with the session ones, not overwritten.
+        session.get(dest.url, headers=HTTP_GET_HEADERS, verify=dest.verify)
+    except (requests.exceptions.ConnectionError,
+            requests.exceptions.ReadTimeout) as e:
         return False, e
     end_time = time.time()
     return True, end_time - start_time
@@ -372,6 +379,14 @@ def run_speedtest(args, conf):
             'even lead to messed up results.',
             conf.getpath('tor', 'control_socket'))
         time.sleep(15)
+
+    # When there will be a refactor where conf is global, this can be removed
+    # from here.
+    state = State(conf.getpath('paths', 'state_fname'))
+    # Call only once to initialize http_headers
+    settings.init_http_headers(conf.get('scanner', 'nickname'), state['uuid'],
+                               str(controller.get_version()))
+
     rl = RelayList(args, conf, controller)
     cb = CB(args, conf, controller, rl)
     rd = ResultDump(args, conf, end_event)
@@ -439,6 +454,9 @@ def main(args, conf):
 
     state = State(conf.getpath('paths', 'state_fname'))
     state['scanner_started'] = now_isodt_str()
+    # Generate an unique identifier for each scanner
+    if 'uuid' not in state:
+        state['uuid'] = str(uuid.uuid4())
 
     try:
         run_speedtest(args, conf)

sbws/globals.py

 import os
 import logging
+import platform
+
+from requests import __version__ as requests_version
+from stem import __version__ as stem_version
+
+from sbws import __version__
+
 
 log = logging.getLogger(__name__)
 
@@ -54,6 +61,32 @@ MAX_BW_DIFF_PERC = 50
 
 BW_LINE_SIZE = 510
 
+# Metadata to send in every requests, so that data servers can know which
+# scanners are using them.
+# In Requests these keys are case insensitive.
+HTTP_HEADERS = {
+    # This would be ignored if changing to HTTP/2
+    'Connection': 'keep-alive',
+    # Needs to get Tor version from the controller
+    'User-Agent': 'sbws/{} ({}) Python/{} Requests/{} Stem/{} Tor/'.format(
+                    __version__, platform.platform(),
+                    platform.python_version(),
+                    requests_version, stem_version),
+    # Organization defined names (:rfc:`7239`)
+    # Needs to get the nickname from the user config file.
+    'Tor-Bandwidth-Scanner-Nickname': '{}',
+    'Tor-Bandwidth-Scanner-UUID': '{}',
+    # In case of including IP address.
+    # 'Forwarded': 'for={}'  # IPv6 part, if there's
+    }
+# In the case of having ipv6 it's concatenated to forwarder.
+IPV6_FORWARDED = ', for="[{}]"'
+
+HTTP_GET_HEADERS = {
+    'Range': '{}',
+    'Accept-Encoding': 'identity',
+}
+
 
 def fail_hard(*a, **kw):
     ''' Log something ... and then exit as fast as possible '''

sbws/lib/destination.py

@@ -73,7 +73,7 @@ def connect_to_destination_over_circuit(dest, circ_id, session, cont, max_dl):
         try:
             # TODO:
             # - What other exceptions can this throw?
-            head = requests_utils.head(session, dest.url, verify=dest.verify)
+            head = session.head(dest.url, verify=dest.verify)
         except (requests.exceptions.ConnectionError,
                 requests.exceptions.ReadTimeout) as e:
             return False, 'Could not connect to {} over circ {} {}: {}'.format(

sbws/util/requests.py

 import requests
+
+from sbws import settings
 import sbws.util.stem as stem_utils
 
 
 def make_session(controller, timeout):
     s = requests.Session()
     socks_info = stem_utils.get_socks_info(controller)
-    s.sbws_proxies = {
+    s.proxies = {
         'http': 'socks5h://{}:{}'.format(*socks_info),
         'https': 'socks5h://{}:{}'.format(*socks_info),
     }
-    s.sbws_timeout = timeout
+    s.timeout = timeout
+    s.headers = settings.HTTP_HEADERS
     return s
-
-
-def get(s, url, **kw):
-    return s.get(url, timeout=s.sbws_timeout, proxies=s.sbws_proxies, **kw)
-
-
-def head(s, url, **kw):
-    return s.head(url, timeout=s.sbws_timeout, proxies=s.sbws_proxies, **kw)