Merge branch 'fix/9874-automate-email-tests' into develop

.travis.requirements.txt

+Babel==0.9.6
+BeautifulSoup==3.2.1
+Mako==0.8.1
+https://ipaddr-py.googlecode.com/files/ipaddr-2.1.10.tar.gz#sha1=c608450b077b19773d4f1b5f1ef88b26f6650ce0#egg=ipaddr-2.1.10-py2.7
+pycrypto==2.6.1
+pygeoip==0.2.7
+pygpgme==0.3
+recaptcha-client==1.0.6
+zope.interface==3.6.1

.travis.yml

@@ -4,11 +4,11 @@ notifications:
   irc:
     channels:
       - "irc.oftc.net#tor-bots"
+    template:
+      - "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}"
+      - "Build details : %{build_url}"
     on_success: always
     on_failure: always
-  template:
-    - "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}"
-    - "Build details : %{build_url}"
   email:
     recipients:
       - isis@torproject.org
@@ -16,17 +16,40 @@ notifications:
     on_success: never
     on_failure: change
 
-before_install:
-  - sudo apt-get update
-  # Distro development header/library/resource/buildtime dependencies
-  - sudo apt-get install --no-install-suggests --no-install-recommends build-essential openssl sqlite3 libgpgme11 libgpgme11-dev python-dev python-setuptools
-
 python:
   - "2.7"
+  - "pypy"
+
+addons:
+  hosts:
+    - bridges.torproject.org
+
+env:
+  global:
+    - LEEKSPIN_VERSION=0.1.3
+    - COVERAGE_VERSION=3.7.1
+    - COVERALLS_VERSION=0.4.2
+    - SURE_VERSION=1.2.2
+  matrix:
+    - TWISTED_VERSION=13.2.0 PYOPENSSL_VERSION=0.13.1
+
+matrix:
+  include:
+    - python: "2.7"
+      env: TWISTED_VERSION=14.0.0 PYOPENSSL_VERSION=0.14
+  allow_failures:
+    - python: "pypy"
+  fast_finish: true
+
+before_install:
+  - sudo apt-get update
 
 install:
-  - pip install -r requirements.txt
-  - pip install --no-use-wheel leekspin==0.1.3 coverage==3.7 coveralls==0.3 sure==1.2.2
+  - sudo apt-get install -qq --no-install-suggests --no-install-recommends build-essential openssl sqlite3 libgpgme11 libgpgme11-dev python-dev python-setuptools
+  - pip install -q -r .travis.requirements.txt
+  - pip install -q --no-use-wheel leekspin==$LEEKSPIN_VERSION sure==$SURE_VERSION
+  - pip install -q --no-use-wheel coverage==$COVERAGE_VERSION coveralls==$COVERALLS_VERSION
+  - pip install -q --no-use-wheel Twisted==$TWISTED_VERSION pyOpenSSL==$PYOPENSSL_VERSION
   - make install
 
 script:

doc/sphinx/source/bridgedb.email.autoresponder.rst

+bridgedb.email.autoresponder
+----------------------------
+
+.. automodule:: bridgedb.email.autoresponder
+    :members:
+    :undoc-members:
+    :private-members:
+    :show-inheritance:

doc/sphinx/source/bridgedb.EmailServer.rst → doc/sphinx/source/bridgedb.email.dkim.rst

-bridgedb.EmailServer
---------------------
+bridgedb.email.dkim
+-------------------
 
-.. automodule:: bridgedb.EmailServer
+.. automodule:: bridgedb.email.dkim
     :members:
     :undoc-members:
     :private-members:

doc/sphinx/source/bridgedb.email.request.rst

+bridgedb.email.request
+----------------------
+
+.. automodule:: bridgedb.email.request
+    :members:
+    :undoc-members:
+    :private-members:
+    :show-inheritance:

doc/sphinx/source/bridgedb.email.server.rst

+bridgedb.email.server
+---------------------
+
+.. automodule:: bridgedb.email.server
+    :members:
+    :undoc-members:
+    :private-members:
+    :show-inheritance:

doc/sphinx/source/bridgedb.email.templates.rst

+bridgedb.email.templates
+------------------------
+
+.. automodule:: bridgedb.email.templates
+    :members:
+    :undoc-members:
+    :private-members:
+    :show-inheritance:

doc/sphinx/source/bridgedb.rst

@@ -11,7 +11,12 @@ BridgeDB Package and Module Documentation
     bridgedb.captcha
     bridgedb.crypto
     bridgedb.Dist
-    bridgedb.EmailServer
+    bridgedb.email
+    bridgedb.email.autoresponder
+    bridgedb.email.dkim
+    bridgedb.email.request
+    bridgedb.email.server
+    bridgedb.email.templates
     bridgedb.Filters
     bridgedb.HTTPServer
     bridgedb.Main

doc/sphinx/source/conf.py

@@ -32,7 +32,12 @@ import bridgedb.Bridges
 import bridgedb.Bucket
 import bridgedb.crypto
 import bridgedb.Dist
-import bridgedb.EmailServer
+import bridgedb.email
+import bridgedb.email.autoresponder
+import bridgedb.email.dkim
+import bridgedb.email.request
+import bridgedb.email.server
+import bridgedb.email.templates
 import bridgedb.Filters
 import bridgedb.HTTPServer
 import bridgedb.Main

lib/bridgedb/email/__init__.py

+"""Servers for BridgeDB's email bridge distributor."""

lib/bridgedb/email/dkim.py

+# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_email_dkim -*-
+#_____________________________________________________________________________
+#
+# This file is part of BridgeDB, a Tor bridge distribution system.
+#
+# :authors: Nick Mathewson <nickm@torproject.org>
+#           Isis Lovecruft <isis@torproject.org> 0xA3ADB67A2CDB8B35
+#           Matthew Finkel <sysrqb@torproject.org>
+#           please also see AUTHORS file
+# :copyright: (c) 2007-2014, The Tor Project, Inc.
+#             (c) 2013-2014, Isis Lovecruft
+# :license: see LICENSE for licensing information
+#_____________________________________________________________________________
+
+"""Functions for checking DKIM verification results in email headers."""
+
+from __future__ import unicode_literals
+
+import logging
+
+
+def checkDKIM(message, rules):
+    """Check the DKIM verification results header.
+
+    This check is only run if the incoming email, **message**, originated from
+    a domain for which we're configured (in the ``EMAIL_DOMAIN_RULES``
+    dictionary in the config file) to check DKIM verification results for.
+
+    :type message: :api:`twisted.mail.smtp.rfc822.Message`
+    :param message: The incoming client request email, including headers.
+    :param dict rules: The list of configured ``EMAIL_DOMAIN_RULES`` for the
+        canonical domain which the client's email request originated from.
+
+    :rtype: bool
+    :returns: ``False`` if:
+        1. We're supposed to expect and check the DKIM headers for the
+           client's email provider domain.
+        2. Those headers were *not* okay.
+        Otherwise, returns ``True``.
+    """
+    logging.info("Checking DKIM verification results...")
+    logging.debug("Domain has rules: %s" % ', '.join(rules))
+
+    if 'dkim' in rules:
+        # getheader() returns the last of a given kind of header; we want
+        # to get the first, so we use getheaders() instead.
+        dkimHeaders = message.getheaders("X-DKIM-Authentication-Results")
+        dkimHeader = "<no header>"
+        if dkimHeaders:
+            dkimHeader = dkimHeaders[0]
+        if not dkimHeader.startswith("pass"):
+            logging.info("Rejecting bad DKIM header on incoming email: %r "
+                         % dkimHeader)
+            return False
+    return True

lib/bridgedb/test/email_helpers.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of BridgeDB, a Tor bridge distribution system.
+#
+# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis@torproject.org>
+#           please also see AUTHORS file
+# :copyright: (c) 2013, Isis Lovecruft
+#             (c) 2007-2013, The Tor Project, Inc.
+#             (c) 2007-2013, all entities within the AUTHORS file
+# :license: see LICENSE for licensing information
+
+
+"""Helpers for testing the email distributor and its servers."""
+
+
+import io
+
+from bridgedb.Dist import IgnoreEmail
+from bridgedb.Dist import TooSoonEmail
+from bridgedb.persistent import Conf
+from bridgedb.email.server import MailServerContext
+from bridgedb.schedule import Unscheduled
+from bridgedb.test.test_HTTPServer import DummyBridge
+
+
+EMAIL_DIST = True
+EMAIL_INCLUDE_FINGERPRINTS = True
+EMAIL_GPG_SIGNING_ENABLED = True
+EMAIL_GPG_SIGNING_KEY = 'TESTING.subkeys.sec'
+EMAIL_DOMAIN_MAP = {
+   'googlemail.com': 'gmail.com',
+   'mail.google.com': 'gmail.com',
+}
+EMAIL_DOMAIN_RULES = {
+   'gmail.com': ["ignore_dots", "dkim"],
+   'example.com': [],
+   'localhost': [],
+}
+EMAIL_DOMAINS = ["gmail.com", "example.com", "localhost"]
+EMAIL_USERNAME = "bridges"
+EMAIL_SMTP_HOST = "127.0.0.1"
+EMAIL_SMTP_PORT = 25
+EMAIL_SMTP_FROM_ADDR = "bridges@localhost"
+EMAIL_N_BRIDGES_PER_ANSWER = 3
+EMAIL_FROM_ADDR = "bridges@localhost"
+EMAIL_BIND_IP = "127.0.0.1"
+EMAIL_PORT = 5225
+
+TEST_CONFIG_FILE = io.StringIO(unicode("""\
+EMAIL_DIST = %s
+EMAIL_INCLUDE_FINGERPRINTS = %s
+EMAIL_GPG_SIGNING_ENABLED = %s
+EMAIL_GPG_SIGNING_KEY = %s
+EMAIL_DOMAIN_MAP = %s
+EMAIL_DOMAIN_RULES = %s
+EMAIL_DOMAINS = %s
+EMAIL_USERNAME = %s
+EMAIL_SMTP_HOST = %s
+EMAIL_SMTP_PORT = %s
+EMAIL_SMTP_FROM_ADDR = %s
+EMAIL_N_BRIDGES_PER_ANSWER = %s
+EMAIL_FROM_ADDR = %s
+EMAIL_BIND_IP = %s
+EMAIL_PORT = %s
+""" % (repr(EMAIL_DIST),
+       repr(EMAIL_INCLUDE_FINGERPRINTS),
+       repr(EMAIL_GPG_SIGNING_ENABLED),
+       repr(EMAIL_GPG_SIGNING_KEY),
+       repr(EMAIL_DOMAIN_MAP),
+       repr(EMAIL_DOMAIN_RULES),
+       repr(EMAIL_DOMAINS),
+       repr(EMAIL_USERNAME),
+       repr(EMAIL_SMTP_HOST),
+       repr(EMAIL_SMTP_PORT),
+       repr(EMAIL_SMTP_FROM_ADDR),
+       repr(EMAIL_N_BRIDGES_PER_ANSWER),
+       repr(EMAIL_FROM_ADDR),
+       repr(EMAIL_BIND_IP),
+       repr(EMAIL_PORT))))
+
+
+def _createConfig(configFile=TEST_CONFIG_FILE):
+    configuration = {}
+    TEST_CONFIG_FILE.seek(0)
+    compiled = compile(configFile.read(), '<string>', 'exec')
+    exec compiled in configuration
+    config = Conf(**configuration)
+    return config
+
+def _createMailServerContext(config=None, distributor=None):
+    if not config:
+        config = _createConfig()
+
+    if not distributor:
+        distributor = DummyEmailDistributor(
+            domainmap=config.EMAIL_DOMAIN_MAP,
+            domainrules=config.EMAIL_DOMAIN_RULES)
+
+    context = MailServerContext(config, distributor, Unscheduled())
+    return context
+
+
+class DummyEmailDistributor(object):
+    """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which is used to
+    test :class:`bridgedb.EmailServer`.
+    """
+
+    def __init__(self, key=None, domainmap=None, domainrules=None,
+                 answerParameters=None):
+        """None of the parameters are really used, ― they are just there to retain an
+        identical method signature.
+        """
+        self.key = self.__class__.__name__
+        self.domainmap = domainmap
+        self.domainrules = domainrules
+        self.answerParameters = answerParameters
+
+    def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None,
+                           countryCode=None, bridgeFilterRules=None):
+        return [DummyBridge() for _ in xrange(N)]
+
+    def cleanDatabase(self):
+        pass
+
+
+class DummyEmailDistributorWithState(DummyEmailDistributor):
+    """A mocked :class:`bridgedb.Dist.EmailBasedDistributor` which raises
+    :exc:`bridgedb.Dist.TooSoonEmail` on the second email and
+    :exc:`bridgedb.Dist.IgnoreEmail` on the third.
+
+    Note that the state tracking is done in a really dumb way. For example, we
+    currently don't consider requests for help text or GnuPG keys to be a
+    "real" request, so in the real email distributor they won't trigger either
+    a TooSoonEmail or IgnoreEmail. Here we only track the total number of
+    *any* type of request per client.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(DummyEmailDistributorWithState, self).__init__()
+        self.alreadySeen = {}
+
+    def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None,
+                           countryCode=None, bridgeFilterRules=None):
+        # Keep track of the number of times we've seen a client.
+        if not emailaddress in self.alreadySeen.keys():
+            self.alreadySeen[emailaddress] = 0
+        self.alreadySeen[emailaddress] += 1
+
+        if self.alreadySeen[emailaddress] <= 1:
+            return [DummyBridge() for _ in xrange(N)]
+        elif self.alreadySeen[emailaddress] == 2:
+            raise TooSoonEmail(
+                "Seen client '%s' %d times"
+                % (emailaddress, self.alreadySeen[emailaddress]),
+                emailaddress)
+        else:
+            raise IgnoreEmail(
+                "Seen client '%s' %d times"
+                % (emailaddress, self.alreadySeen[emailaddress]),
+                emailaddress)

lib/bridgedb/test/test_email_dkim.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of BridgeDB, a Tor bridge distribution system.
+#
+# :authors: Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis@torproject.org>
+#           please also see AUTHORS file
+# :copyright: (c) 2013, Isis Lovecruft
+#             (c) 2007-2013, The Tor Project, Inc.
+#             (c) 2007-2013, all entities within the AUTHORS file
+# :license: 3-Clause BSD, see LICENSE for licensing information
+
+"""Unittests for the :mod:`bridgedb.email.dkim` module."""
+
+import io
+
+from twisted.mail.smtp import rfc822
+from twisted.trial import unittest
+
+from bridgedb.email import dkim
+
+
+class CheckDKIMTests(unittest.TestCase):
+    """Tests for :func:`email.server.checkDKIM`."""
+
+    def setUp(self):
+        """Create fake email, distributor, and associated context data."""
+        self.goodMessage = """\
+From: user@gmail.com
+To: bridges@localhost
+X-DKIM-Authentication-Results: pass
+Subject: testing
+
+get bridges
+"""
+        self.badMessage = """\
+From: user@gmail.com
+To: bridges@localhost
+Subject: testing
+
+get bridges
+"""
+        self.domainRules = {
+            'gmail.com': ["ignore_dots", "dkim"],
+            'example.com': [],
+            'localhost': [],
+        }
+
+    def _createMessage(self, messageString):
+        """Create an ``rfc822.Message`` from a string."""
+        messageIO = io.StringIO(unicode(messageString))
+        return rfc822.Message(messageIO)
+
+    def test_checkDKIM_good(self):
+        message = self._createMessage(self.goodMessage)
+        result = dkim.checkDKIM(message,
+                                self.domainRules.get("gmail.com"))
+        self.assertTrue(result)
+                                         
+
+    def test_checkDKIM_bad(self):
+        message = self._createMessage(self.badMessage)
+        result = dkim.checkDKIM(message,
+                                self.domainRules.get("gmail.com"))
+        self.assertIs(result, False)
+
+    def test_checkDKIM_dunno(self):
+        """A ``X-DKIM-Authentication-Results: dunno`` header should return
+        False.
+        """
+        messageList = self.badMessage.split('\n')
+        messageList[2] = "X-DKIM-Authentication-Results: dunno"
+        message = self._createMessage('\n'.join(messageList))
+        result = dkim.checkDKIM(message,
+                                self.domainRules.get("gmail.com"))
+        self.assertIs(result, False)
+
+    def test_checkDKIM_good_dunno(self):
+        """A good DKIM verification header, *plus* an
+        ``X-DKIM-Authentication-Results: dunno`` header should return False.
+        """
+        messageList = self.badMessage.split('\n')
+        messageList.insert(2, "X-DKIM-Authentication-Results: dunno")
+        message = self._createMessage('\n'.join(messageList))
+        result = dkim.checkDKIM(message,
+                                self.domainRules.get("gmail.com"))
+        self.assertIs(result, False)

lib/bridgedb/test/util.py

@@ -14,11 +14,14 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import abc
 import doctest
 import os
 
 from functools import wraps
 
+from twisted.trial import unittest
+
 
 def fileCheckDecorator(func):
     """Method decorator for a t.t.unittest.TestCase test_* method.
@@ -62,5 +65,12 @@ def fileCheckDecorator(func):
                         % (str(description), dst, src))
     return wrapper
 
+
+class TestCaseMixin:
+    """Subclasses of me can be used as mix-in classes with ``TestCase``s."""
+    __metaclass__ = abc.ABCMeta
+TestCaseMixin.register(unittest.TestCase)
+
+
 if __name__ == "__main__":
     doctest.run_docstring_examples(fileCheckDecorator, None)

requirements.txt

 Babel==0.9.6
 BeautifulSoup==3.2.1
 Mako==0.8.1
-Twisted==13.1.0
+Twisted==13.2.0
 https://ipaddr-py.googlecode.com/files/ipaddr-2.1.10.tar.gz#sha1=c608450b077b19773d4f1b5f1ef88b26f6650ce0#egg=ipaddr-2.1.10-py2.7
 pycrypto==2.6.1
 pyOpenSSL==0.13.1
 pygeoip==0.2.7
 pygpgme==0.3
 recaptcha-client==1.0.6
-zope.interface>=4.0.5
+zope.interface==3.6.1

setup.py

@@ -284,7 +284,7 @@ setuptools.setup(
               'bridgedb.test'],
     scripts=['scripts/bridgedb'],
     extras_require={'test': ["sure==1.2.2",
-                             "coverage==3.7",
+                             "coverage==3.7.1",
                              "leekspin==0.1.3"]},
     zip_safe=False,
     cmdclass=get_cmdclass(),