Unverified Commit 9a7ae57c authored by Philipp Winter's avatar Philipp Winter
Browse files

Merge branch 'release-0.9.4'

parents 0f2349d1 e461a7a0
......@@ -15,7 +15,7 @@
#------------------------------------------------------------------------------
attrs==19.2.0
Babel==2.8.0
BeautifulSoup==3.2.2
beautifulsoup4==4.8.2
Mako==1.1.1
pycryptodome==3.9.6
Twisted==19.10.0
......@@ -25,6 +25,7 @@ gnupg==2.3.1
ipaddr==2.2.0
mechanize==0.4.5
Pillow==6.2.2
pyOpenSSL==19.0.0
pygeoip==0.3.2
qrcode==6.1
service_identity==18.1.0
......
......@@ -16,7 +16,7 @@ notifications:
on_failure: change
python:
- "2.7"
- "3.7"
addons:
hosts:
......
Changes in version 0.9.4 - 2020-02-19
* FIXES https://bugs.torproject.org/30946
This patch ports BridgeDB to Python 3. Python 2 is no longer supported
since Jan 1, 2020.
Changes in version 0.9.3 - 2020-02-18
* FIXES <https://bugs.torproject.org/33299>
......
include versioneer.py
include bridgedb/_version.py
include requirements.txt
recursice-include bridgedb/i18n *.po *.pot
recursive-include bridgedb/i18n *.po *.pot
......@@ -10,6 +10,7 @@
them into hashrings for distributors.
"""
import binascii
import bisect
import logging
import re
......@@ -28,12 +29,6 @@ from bridgedb.parse.fingerprint import isValidFingerprint
from bridgedb.parse.fingerprint import toHex
from bridgedb.safelog import logSafely
try:
from cStringIO import StringIO
except ImportError:
from io import StringIO
ID_LEN = 20 # XXX Only used in commented out line in Storage.py
DIGEST_LEN = 20
PORTSPEC_LEN = 16
......@@ -335,7 +330,7 @@ class BridgeRing(object):
else:
logging.debug(
"Got duplicate bridge %r in main hashring for position %r."
% (logSafely(k.encode('hex')), pos.encode('hex')))
% (logSafely(binascii.hexlify(k).decode('utf-8')), binascii.hexlify(pos).decode('utf-8')))
keys.sort()
if filterBySubnet:
......@@ -361,7 +356,7 @@ class BridgeRing(object):
def dumpAssignments(self, f, description=""):
logging.info("Dumping bridge assignments for %s..." % self.name)
for b in self.bridges.itervalues():
for b in self.bridges.values():
desc = [ description ]
for tp,val,_,subring in self.subrings:
if subring.getBridgeByID(b.identity):
......@@ -380,7 +375,7 @@ class FixedBridgeSplitter(object):
def insert(self, bridge):
# Grab the first 4 bytes
digest = self.hmac(bridge.identity)
pos = long( digest[:8], 16 )
pos = int( digest[:8], 16 )
which = pos % len(self.rings)
self.rings[which].insert(bridge)
......@@ -405,7 +400,7 @@ class FixedBridgeSplitter(object):
description is ``"IPv6 obfs2 bridges"`` the line would read:
``"IPv6 obfs2 bridges ring=3"``.
"""
for index, ring in zip(xrange(len(self.rings)), self.rings):
for index, ring in zip(range(len(self.rings)), self.rings):
ring.dumpAssignments(filename, "%s ring=%s" % (description, index))
......@@ -544,7 +539,7 @@ class BridgeSplitter(object):
logging.info("Current rings: %s" % " ".join(self.ringsByName))
def dumpAssignments(self, f, description=""):
for name,ring in self.ringsByName.iteritems():
for name,ring in self.ringsByName.items():
ring.dumpAssignments(f, "%s %s" % (description, name))
......@@ -633,8 +628,8 @@ class FilteredBridgeSplitter(object):
"""
filterNames = []
for filterName in [x.func_name for x in list(ringname)]:
# Using `assignBridgesToSubring.func_name` gives us a messy
for filterName in [x.__name__ for x in list(ringname)]:
# Using `assignBridgesToSubring.__name__` gives us a messy
# string which includes all parameters and memory addresses. Get
# rid of this by partitioning at the first `(`:
realFilterName = filterName.partition('(')[0]
......
......@@ -33,7 +33,7 @@ from bridgedb.schedule import toUnixSeconds
# tunables
weighting_factor = float(19)/float(20)
discountIntervalMillis = long(60*60*12*1000)
discountIntervalMillis = 60*60*12*1000
class BridgeHistory(object):
......@@ -76,15 +76,15 @@ class BridgeHistory(object):
self.fingerprint = fingerprint
self.ip = ip
self.port = port
self.weightedUptime = long(weightedUptime)
self.weightedTime = long(weightedTime)
self.weightedRunLength = long(weightedRunLength)
self.weightedUptime = int(weightedUptime)
self.weightedTime = int(weightedTime)
self.weightedRunLength = int(weightedRunLength)
self.totalRunWeights = float(totalRunWeights)
self.lastSeenWithDifferentAddressAndPort = \
long(lastSeenWithDifferentAddressAndPort)
self.lastSeenWithThisAddressAndPort = long(lastSeenWithThisAddressAndPort)
self.lastDiscountedHistoryValues = long(lastDiscountedHistoryValues)
self.lastUpdatedWeightedTime = long(lastUpdatedWeightedTime)
int(lastSeenWithDifferentAddressAndPort)
self.lastSeenWithThisAddressAndPort = int(lastSeenWithThisAddressAndPort)
self.lastDiscountedHistoryValues = int(lastDiscountedHistoryValues)
self.lastUpdatedWeightedTime = int(lastUpdatedWeightedTime)
def discountWeightedFractionalUptimeAndWeightedTime(self, discountUntilMillis):
""" discount weighted times """
......@@ -111,8 +111,8 @@ class BridgeHistory(object):
@property
def weightedFractionalUptime(self):
"""Weighted Fractional Uptime"""
if self.weightedTime <0.0001: return long(0)
return long(10000) * self.weightedUptime / self.weightedTime
if self.weightedTime <0.0001: return 0
return 10000 * self.weightedUptime / self.weightedTime
@property
def tosa(self):
......@@ -127,7 +127,7 @@ class BridgeHistory(object):
more recently than it, or if it has been around for a Weighted Time of 8 days.
"""
# if this bridge has been around longer than 8 days
if self.weightedTime >= long(8 * 24 * 60 * 60):
if self.weightedTime >= 8 * 24 * 60 * 60:
return True
# return True if self.weightedTime is greater than the weightedTime
......@@ -146,10 +146,10 @@ class BridgeHistory(object):
"""Weighted Mean Time Between Address Change"""
totalRunLength = self.weightedRunLength + \
((self.lastSeenWithThisAddressAndPort -
self.lastSeenWithDifferentAddressAndPort) / long(1000))
self.lastSeenWithDifferentAddressAndPort) / 1000)
totalWeights = self.totalRunWeights + 1.0
if totalWeights < 0.0001: return long(0)
if totalWeights < 0.0001: return 0
assert(isinstance(long,totalRunLength))
assert(isinstance(long,totalWeights))
return totalRunlength / totalWeights
......@@ -159,9 +159,9 @@ def addOrUpdateBridgeHistory(bridge, timestamp):
bhe = db.getBridgeHistory(bridge.fingerprint)
if not bhe:
# This is the first status, assume 60 minutes.
secondsSinceLastStatusPublication = long(60*60)
lastSeenWithDifferentAddressAndPort = timestamp * long(1000)
lastSeenWithThisAddressAndPort = timestamp * long(1000)
secondsSinceLastStatusPublication = 60*60
lastSeenWithDifferentAddressAndPort = timestamp * 1000
lastSeenWithThisAddressAndPort = timestamp * 1000
bhe = BridgeHistory(
bridge.fingerprint, bridge.address, bridge.orPort,
......@@ -179,9 +179,9 @@ def addOrUpdateBridgeHistory(bridge, timestamp):
# Calculate the seconds since the last parsed status. If this is
# the first status or we haven't seen a status for more than 60
# minutes, assume 60 minutes.
statusPublicationMillis = long(timestamp * 1000)
statusPublicationMillis = timestamp * 1000
if (statusPublicationMillis - bhe.lastSeenWithThisAddressAndPort) > 60*60*1000:
secondsSinceLastStatusPublication = long(60*60)
secondsSinceLastStatusPublication = 60*60
logging.debug("Capping secondsSinceLastStatusPublication to 1 hour")
# otherwise, roll with it
else:
......@@ -278,7 +278,7 @@ def updateBridgeHistory(bridges, timestamps):
logging.debug("Beginning bridge stability calculations")
sortedTimestamps = {}
for fingerprint, stamps in timestamps.items()[:]:
for fingerprint, stamps in timestamps.items():
stamps.sort()
bridge = bridges[fingerprint]
for timestamp in stamps:
......
......@@ -8,7 +8,6 @@ import binascii
import sqlite3
import time
import hashlib
from contextlib import GeneratorContextManager
from functools import wraps
from ipaddr import IPAddress
import sys
......@@ -212,7 +211,7 @@ class Database(object):
cur.execute("DELETE FROM EmailedBridges WHERE when_mailed < ?", (t,))
def getEmailTime(self, addr):
addr = hashlib.sha1(addr).hexdigest()
addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
cur.execute("SELECT when_mailed FROM EmailedBridges WHERE email = ?", (addr,))
v = cur.fetchone()
......@@ -221,7 +220,7 @@ class Database(object):
return strToTime(v[0])
def setEmailTime(self, addr, whenMailed):
addr = hashlib.sha1(addr).hexdigest()
addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
t = timeToStr(whenMailed)
cur.execute("INSERT OR REPLACE INTO EmailedBridges "
......@@ -262,7 +261,7 @@ class Database(object):
(distributor, hex_key))
def getWarnedEmail(self, addr):
addr = hashlib.sha1(addr).hexdigest()
addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
cur = self._cur
cur.execute("SELECT * FROM WarnedEmails WHERE email = ?", (addr,))
v = cur.fetchone()
......@@ -271,7 +270,7 @@ class Database(object):
return True
def setWarnedEmail(self, addr, warned=True, whenWarned=time.time()):
addr = hashlib.sha1(addr).hexdigest()
addr = hashlib.sha1(addr.encode('utf-8')).hexdigest()
t = timeToStr(whenWarned)
cur = self._cur
if warned == True:
......@@ -345,11 +344,18 @@ def openDatabase(sqlite_file):
return conn
class DBGeneratorContextManager(GeneratorContextManager):
class DBGeneratorContextManager(object):
"""Helper for @contextmanager decorator.
Overload __exit__() so we can call the generator many times
"""
def __init__(self, gen):
self.gen = gen
def __enter__(self):
return next(self.gen)
def __exit__(self, type, value, traceback):
"""Handle exiting a with statement block
......@@ -362,7 +368,7 @@ class DBGeneratorContextManager(GeneratorContextManager):
"""
if type is None:
try:
self.gen.next()
next(self.gen)
except StopIteration:
return
return
......@@ -374,7 +380,7 @@ class DBGeneratorContextManager(GeneratorContextManager):
try:
self.gen.throw(type, value, traceback)
raise RuntimeError("generator didn't stop after throw()")
except StopIteration, exc:
except StopIteration as exc:
# Suppress the exception *unless* it's the same exception that
# was passed to throw(). This prevents a StopIteration
# raised inside the "with" statement from being suppressed
......
......@@ -164,7 +164,7 @@ class BridgeRequestBase(object):
# Get an HMAC with the key of the client identifier:
digest = getHMACFunc(key)(client)
# Take the lower 8 bytes of the digest and convert to a long:
position = long(digest[:8], 16)
position = int(digest[:8], 16)
return position
def isValid(self, valid=None):
......
......@@ -240,7 +240,7 @@ class BridgeAddressBase(object):
:param str value: The binary-encoded SHA-1 hash digest of the public
half of this Bridge's identity key.
"""
self.fingerprint = toHex(value)
self.fingerprint = toHex(value).decode('utf-8')
@identity.deleter
def identity(self):
......@@ -743,7 +743,7 @@ class BridgeBackwardsCompatibility(BridgeBase):
if not fingerprint:
if not len(idDigest) == 20:
raise TypeError("Bridge with invalid ID")
self.fingerprint = toHex(idDigest)
self.fingerprint = toHex(idDigest).decode('utf-8')
elif fingerprint:
if not isValidFingerprint(fingerprint):
raise TypeError("Bridge with invalid fingerprint (%r)"
......@@ -1037,7 +1037,7 @@ class Bridge(BridgeBackwardsCompatibility):
if safelog.safe_logging:
prefix = '$$'
if fingerprint:
fingerprint = hashlib.sha1(fingerprint).hexdigest().upper()
fingerprint = hashlib.sha1(fingerprint.encode('utf-8')).hexdigest().upper()
if not fingerprint:
fingerprint = '0' * 40
......@@ -1177,7 +1177,7 @@ class Bridge(BridgeBackwardsCompatibility):
# their ``methodname`` matches the requested transport:
transports = filter(lambda pt: pt.methodname == desired, self.transports)
# Filter again for whichever of IPv4 or IPv6 was requested:
transports = filter(lambda pt: pt.address.version == ipVersion, transports)
transports = list(filter(lambda pt: pt.address.version == ipVersion, transports))
if not transports:
raise PluggableTransportUnavailable(
......@@ -1377,7 +1377,7 @@ class Bridge(BridgeBackwardsCompatibility):
:meth:`_getBlockKey`.
:param str countryCode: A two-character country code specifier.
"""
if self._blockedIn.has_key(key):
if key in self._blockedIn:
self._blockedIn[key].append(countryCode.lower())
else:
self._blockedIn[key] = [countryCode.lower(),]
......@@ -1665,7 +1665,7 @@ class Bridge(BridgeBackwardsCompatibility):
logging.info("Verifying extrainfo signature for %s..." % self)
# Get the bytes of the descriptor signature without the headers:
document, signature = descriptor.get_bytes().split(TOR_BEGIN_SIGNATURE)
document, signature = str(descriptor).split(TOR_BEGIN_SIGNATURE)
signature = signature.replace(TOR_END_SIGNATURE, '')
signature = signature.replace('\n', '')
signature = signature.strip()
......@@ -1709,8 +1709,8 @@ class Bridge(BridgeBackwardsCompatibility):
# This is the hexadecimal SHA-1 hash digest of the descriptor document
# as it was signed:
signedDigest = codecs.encode(unpadded, 'hex_codec')
actualDigest = hashlib.sha1(document).hexdigest()
signedDigest = codecs.encode(unpadded, 'hex_codec').decode('utf-8')
actualDigest = hashlib.sha1(document.encode('utf-8')).hexdigest()
except Exception as error:
logging.debug("Error verifying extrainfo signature: %s" % error)
......
......@@ -63,11 +63,10 @@ import logging
import random
import os
import time
import urllib2
import urllib.request
from BeautifulSoup import BeautifulSoup
from zope.interface import Interface, Attribute, implements
from bs4 import BeautifulSoup
from zope.interface import Interface, Attribute, implementer
from bridgedb import crypto
from bridgedb import schedule
......@@ -101,6 +100,7 @@ class ICaptcha(Interface):
"""Retrieve a new CAPTCHA image."""
@implementer(ICaptcha)
class Captcha(object):
"""A generic CAPTCHA base class.
......@@ -117,7 +117,6 @@ class Captcha(object):
:ivar secretKey: A private key used for decrypting challenge strings during
CAPTCHA solution verification.
"""
implements(ICaptcha)
def __init__(self, publicKey=None, secretKey=None):
"""Obtain a new CAPTCHA for a client."""
......@@ -185,14 +184,14 @@ class ReCaptcha(Captcha):
form = "/noscript?k=%s" % self.publicKey
# Extract and store image from recaptcha
html = urllib2.urlopen(urlbase + form).read()
html = urllib.request.urlopen(urlbase + form).read()
# FIXME: The remaining lines currently cannot be reliably unit tested:
soup = BeautifulSoup(html) # pragma: no cover
imgurl = urlbase + "/" + soup.find('img')['src'] # pragma: no cover
cField = soup.find( # pragma: no cover
'input', {'name': 'recaptcha_challenge_field'}) # pragma: no cover
self.challenge = str(cField['value']) # pragma: no cover
self.image = urllib2.urlopen(imgurl).read() # pragma: no cover
self.image = urllib.request.urlopen(imgurl).read() # pragma: no cover
class GimpCaptcha(Captcha):
......@@ -271,6 +270,10 @@ class GimpCaptcha(Captcha):
:returns: ``True`` if the CAPTCHA solution was correct and not
stale. ``False`` otherwise.
"""
if isinstance(solution, bytes):
solution = solution.decode('utf-8')
hmacIsValid = False
if not solution:
......@@ -290,12 +293,12 @@ class GimpCaptcha(Captcha):
if hmacIsValid:
try:
answerBlob = secretKey.decrypt(encBlob)
timestamp = answerBlob[:12].lstrip('0')
timestamp = answerBlob[:12].lstrip(b'0')
then = cls.sched.nextIntervalStarts(int(timestamp))
now = int(time.time())
answer = answerBlob[12:]
answer = answerBlob[12:].decode('utf-8')
except Exception as error:
logging.warn(error.message)
logging.warn(str(error))
else:
# If the beginning of the 'next' interval (the interval
# after the one when the CAPTCHA timestamp was created)
......@@ -367,10 +370,10 @@ class GimpCaptcha(Captcha):
"""
timestamp = str(int(time.time())).zfill(12)
blob = timestamp + answer
encBlob = self.publicKey.encrypt(blob)
encBlob = self.publicKey.encrypt(blob.encode('utf-8'))
hmac = crypto.getHMAC(self.hmacKey, encBlob)
challenge = urlsafe_b64encode(hmac + encBlob)
return challenge
return challenge.decode("utf-8")
def get(self):
"""Get a random CAPTCHA from the cache directory.
......@@ -388,7 +391,7 @@ class GimpCaptcha(Captcha):
try:
imageFilename = random.choice(os.listdir(self.cacheDir))
imagePath = os.path.join(self.cacheDir, imageFilename)
with open(imagePath) as imageFile:
with open(imagePath, 'rb') as imageFile:
self.image = imageFile.read()
except IndexError:
raise GimpCaptchaError("CAPTCHA cache dir appears empty: %r"
......
......@@ -82,7 +82,7 @@ def loadConfig(configFile=None, configCls=None):
if itsSafeToUseLogging:
logging.info("Loading settings from config file: '%s'" % conffile)
compiled = compile(open(conffile).read(), '<string>', 'exec')
exec compiled in configuration
exec(compiled, configuration)
if itsSafeToUseLogging:
logging.debug("New configuration settings:")
......
......@@ -47,7 +47,7 @@ import io
import logging
import os
import re
import urllib
import urllib.parse
import OpenSSL
......@@ -57,29 +57,9 @@ from Crypto.PublicKey import RSA
from twisted.internet import ssl
from twisted.python.procutils import which
#: The hash digest to use for HMACs.
DIGESTMOD = hashlib.sha1
# Test to see if we have the old or new style buffer() interface. Trying
# to use an old-style buffer on Python2.7 prior to version 2.7.5 will produce:
#
# TypeError: 'buffer' does not have the buffer interface
#
#: ``True`` if we have the new-style `buffer`_ interface; ``False`` otherwise.
#:
#: .. _buffer: https://docs.python.org/2/c-api/buffer.html
NEW_BUFFER_INTERFACE = False
try:
io.BytesIO(buffer('test'))
except TypeError: # pragma: no cover
logging.warn(
"This Python version is too old! "\
"It doesn't support new-style buffer interfaces: "\
"https://mail.python.org/pipermail/python-dev/2010-October/104917.html")
else:
NEW_BUFFER_INTERFACE = True
class PKCS1PaddingError(Exception):
"""Raised when there is a problem adding or removing PKCS#1 padding."""
......@@ -89,7 +69,7 @@ class RSAKeyGenerationError(Exception):
def writeKeyToFile(key, filename):
"""Write **key** to **filename**, with ``0400`` permissions.
"""Write **key** to **filename**, with ``400`` octal permissions.
If **filename** doesn't exist, it will be created. If it does exist
already, and is writable by the owner of the current process, then it will
......@@ -102,7 +82,7 @@ def writeKeyToFile(key, filename):
"""
logging.info("Writing key to file: %r" % filename)
flags = os.O_WRONLY | os.O_TRUNC | os.O_CREAT | getattr(os, "O_BIN", 0)
fd = os.open(filename, flags, 0400)
fd = os.open(filename, flags, 0o400)
os.write(fd, key)
os.fsync(fd)
os.close(fd)
......@@ -210,6 +190,12 @@ def getKey(filename):
def getHMAC(key, value):
"""Return the HMAC of **value** using the **key**."""
# normalize inputs to be bytes
key = key.encode('utf-8') if isinstance(key, str) else key
value = value.encode('utf-8') if isinstance(value, str) else value
h = hmac.new(key, value, digestmod=DIGESTMOD)
return h.digest()
......@@ -220,14 +206,19 @@ def getHMACFunc(key, hex=True):
:rtype: callable
:returns: A function which can be uses to generate HMACs.
"""
key = key.encode('utf-8') if isinstance(key, str) else key
h = hmac.new(key, digestmod=DIGESTMOD)
def hmac_fn(value):
value = value.encode('utf-8') if isinstance(value, str) else value
h_tmp = h.copy()
h_tmp.update(value)
if hex:
return h_tmp.hexdigest()
else:
return h_tmp.digest()
return hmac_fn
def removePKCS1Padding(message):
......@@ -319,11 +310,12 @@ def initializeGnuPG(config):
logging.warn("No secret keys found in %s!" % gpg.secring)
return ret
primarySK = filter(lambda key: key['fingerprint'] == primary, secrets)
primaryPK = filter(lambda key: key['fingerprint'] == primary, publics)
primarySK = list(filter(lambda key: key['fingerprint'] == primary, secrets))
primaryPK = list(filter(lambda key: key['fingerprint'] == primary, publics))
if primarySK and primaryPK:
logging.info("Found GnuPG primary key with fingerprint: %s" % primary)
for sub in primaryPK[0]['subkeys']:
logging.info(" Subkey: %s Usage: %s" % (sub[0], sub[1].upper()))
else:
......@@ -419,7 +411,8 @@ class SSLVerifyingContextFactory(ssl.CertificateOptions):
:rtype: str
:returns: The full hostname (including any subdomains).
"""
hostname = urllib.splithost(urllib.splittype(url)[1])[0]
hostname = urllib.parse.urlparse(url).netloc
logging.debug("Parsed hostname %r for cert CN matching." % hostname)
return hostname
......
......@@ -108,7 +108,7 @@ import math
from zope import interface
from zope.interface import Attribute
from zope.interface import implements
from zope.interface import implementer
# from bridgedb.hashring import IHashring
from bridgedb.interfaces import IName
......@@ -155,12 +155,12 @@ class IDistribute(IName):
"""Get bridges based on a client's **bridgeRequest**."""
@implementer(IDistribute)
class Distributor(Named):
"""A :class:`Distributor` distributes bridges to clients.
Inherit from me to create a new type of ``Distributor``.
"""
implements(IDistribute)
_bridgesPerResponseMin = 1
_bridgesPerResponseMax = 3
......
......@@ -16,9 +16,3 @@
distributor.
'''
import autoresponder
import distributor
import dkim
import request