Commit 58c45eab authored by Hiro's avatar Hiro 🏄
Browse files

Refactor email and add tests for email and locales support

parent 1089ebf1
{ {
"platforms": ["linux", "osx", "windows"], "platforms": ["linux", "osx", "windows"],
"languages": ["en", "es", "pt"],
"dbname": "/srv/gettor.torproject.org/home/gettor/gettor.db", "dbname": "/srv/gettor.torproject.org/home/gettor/gettor.db",
"email_parser_logfile": "/srv/gettor.torproject.org/home/gettor/log/email_parser.log", "email_parser_logfile": "/srv/gettor.torproject.org/home/gettor/log/email_parser.log",
"email_requests_limit": 5, "email_requests_limit": 5,
......
...@@ -28,7 +28,7 @@ from twisted.internet import defer ...@@ -28,7 +28,7 @@ from twisted.internet import defer
from twisted.enterprise import adbapi from twisted.enterprise import adbapi
from ..utils.db import SQLite3 from ..utils.db import SQLite3
from ..utils import strings
class AddressError(Exception): class AddressError(Exception):
""" """
...@@ -57,26 +57,7 @@ class EmailParser(object): ...@@ -57,26 +57,7 @@ class EmailParser(object):
self.dkim = dkim self.dkim = dkim
self.to_addr = to_addr self.to_addr = to_addr
def normalize(self, msg):
def parse(self, msg_str):
"""
Parse message content. Check if email address is well formed, if DKIM
signature is valid, and prevent service flooding. Finally, look for
commands to process the request. Current commands are:
- links: request links for download.
- help: help request.
:param msg_str (str): incomming message as string.
:return dict with email address and command (`links` or `help`).
"""
platforms = self.settings.get("platforms")
languages = self.settings.get("languages")
log.msg("Building email message from string.", system="email parser")
msg = message_from_string(msg_str)
# Normalization will convert <Alice Wonderland> alice@wonderland.net # Normalization will convert <Alice Wonderland> alice@wonderland.net
# into alice@wonderland.net # into alice@wonderland.net
name, norm_addr = parseaddr(msg['From']) name, norm_addr = parseaddr(msg['From'])
...@@ -85,7 +66,10 @@ class EmailParser(object): ...@@ -85,7 +66,10 @@ class EmailParser(object):
"Normalizing and validating FROM email address.", "Normalizing and validating FROM email address.",
system="email parser" system="email parser"
) )
return name, norm_addr, to_name, norm_to_addr
def validate(self, norm_addr, msg):
# Validate_email will do a bunch of regexp to see if the email address # Validate_email will do a bunch of regexp to see if the email address
# is well address. Additional options for validate_email are check_mx # is well address. Additional options for validate_email are check_mx
# and verify, which check if the SMTP host and email address exist. # and verify, which check if the SMTP host and email address exist.
...@@ -95,6 +79,8 @@ class EmailParser(object): ...@@ -95,6 +79,8 @@ class EmailParser(object):
"Email address normalized and validated.", "Email address normalized and validated.",
system="email parser" system="email parser"
) )
return True
else: else:
log.err( log.err(
"Error normalizing/validating email address.", "Error normalizing/validating email address.",
...@@ -102,17 +88,8 @@ class EmailParser(object): ...@@ -102,17 +88,8 @@ class EmailParser(object):
) )
raise AddressError("Invalid email address {}".format(msg['From'])) raise AddressError("Invalid email address {}".format(msg['From']))
hid = hashlib.sha256(norm_addr.encode('utf-8'))
log.msg(
"Request from {}".format(hid.hexdigest()), system="email parser"
)
if self.to_addr:
if self.to_addr != norm_to_addr:
log.msg("Got request for a different instance of gettor")
log.msg("Intended recipient: {}".format(norm_to_addr))
return {}
def dkim_verify(self, msg_str, norm_addr):
# DKIM verification. Simply check that the server has verified the # DKIM verification. Simply check that the server has verified the
# message's signature # message's signature
if self.dkim: if self.dkim:
...@@ -121,6 +98,7 @@ class EmailParser(object): ...@@ -121,6 +98,7 @@ class EmailParser(object):
# string, so DKIM will fail. Use the original string instead # string, so DKIM will fail. Use the original string instead
if dkim.verify(msg_str): if dkim.verify(msg_str):
log.msg("Valid DKIM signature.", system="email parser") log.msg("Valid DKIM signature.", system="email parser")
return True
else: else:
log.msg("Invalid DKIM signature.", system="email parser") log.msg("Invalid DKIM signature.", system="email parser")
username, domain = norm_addr.split("@") username, domain = norm_addr.split("@")
...@@ -129,7 +107,12 @@ class EmailParser(object): ...@@ -129,7 +107,12 @@ class EmailParser(object):
hid.hexdigest(), domain hid.hexdigest(), domain
) )
) )
# Is this even useful like this?
else:
return True
def build_request(self, msg_str, norm_addr, languages, platforms):
# Search for commands keywords # Search for commands keywords
subject_re = re.compile(r"Subject: (.*)\r\n") subject_re = re.compile(r"Subject: (.*)\r\n")
subject = subject_re.search(msg_str) subject = subject_re.search(msg_str)
...@@ -167,6 +150,54 @@ class EmailParser(object): ...@@ -167,6 +150,54 @@ class EmailParser(object):
return request return request
def parse(self, msg_str):
"""
Parse message content. Check if email address is well formed, if DKIM
signature is valid, and prevent service flooding. Finally, look for
commands to process the request. Current commands are:
- links: request links for download.
- help: help request.
:param msg_str (str): incomming message as string.
:return dict with email address and command (`links` or `help`).
"""
log.msg("Building email message from string.", system="email parser")
platforms = self.settings.get("platforms")
languages = [*strings.get_locales().keys()]
msg = message_from_string(msg_str)
name, norm_addr, to_name, norm_to_addr = self.normalize(msg)
try:
self.validate(norm_addr, msg)
except AddressError as e:
log.message("Address error: {}".format(e.args))
hid = hashlib.sha256(norm_addr.encode('utf-8'))
log.msg(
"Request from {}".format(hid.hexdigest()), system="email parser"
)
if self.to_addr:
if self.to_addr != norm_to_addr:
log.msg("Got request for a different instance of gettor")
log.msg("Intended recipient: {}".format(norm_to_addr))
return {}
try:
self.dkim_verify(msg_str, norm_addr)
except ValueError as e:
log.msg("DKIM error: {}".format(e.args))
request = self.build_request(msg_str, norm_addr, languages, platforms)
return request
@defer.inlineCallbacks @defer.inlineCallbacks
def parse_callback(self, request): def parse_callback(self, request):
""" """
......
...@@ -124,7 +124,7 @@ class Sendmail(object): ...@@ -124,7 +124,7 @@ class Sendmail(object):
for request in help_requests: for request in help_requests:
id = request[0] id = request[0]
date = request[4] date = request[5]
hid = hashlib.sha256(id.encode('utf-8')) hid = hashlib.sha256(id.encode('utf-8'))
log.info( log.info(
...@@ -164,11 +164,10 @@ class Sendmail(object): ...@@ -164,11 +164,10 @@ class Sendmail(object):
if not language: if not language:
language = 'en' language = 'en'
locales = { 'en': 'en-US', locales = strings.get_locales()
'es': 'es-ES',
'pt': 'pt-BR'}
strings.load_strings(language) strings.load_strings(language)
locale = locales[language] locale = locales[language]['locale']
log.info("Getting links for {}.".format(platform)) log.info("Getting links for {}.".format(platform))
links = yield self.conn.get_links( links = yield self.conn.get_links(
......
...@@ -59,7 +59,6 @@ class Settings(object): ...@@ -59,7 +59,6 @@ class Settings(object):
else: else:
self._settings = { self._settings = {
"platforms": ["linux", "osx", "windows"], "platforms": ["linux", "osx", "windows"],
"languages": ["en", "es", "pt"],
"dbname": "/srv/gettor.torproject.org/home/gettor/gettor.db", "dbname": "/srv/gettor.torproject.org/home/gettor/gettor.db",
"email_parser_logfile": "/srv/gettor.torproject.org/home/gettor/log/email_parser.log", "email_parser_logfile": "/srv/gettor.torproject.org/home/gettor/log/email_parser.log",
"email_requests_limit": 5, "email_requests_limit": 5,
......
{ {
"en": "English", "en": {
"es": "Español", "language": "English",
"pt": "Português Brasil" "locale": "en-US"
},
"es": {
"language": "Español",
"locale": "es-ES"
},
"pt": {
"language": "Português Brasil",
"locale": "pt-BR"
}
} }
...@@ -6,3 +6,6 @@ from gettor.utils import options ...@@ -6,3 +6,6 @@ from gettor.utils import options
from gettor.utils import strings from gettor.utils import strings
from gettor.services.email import sendmail from gettor.services.email import sendmail
from gettor.parse.email import EmailParser, AddressError, DKIMError from gettor.parse.email import EmailParser, AddressError, DKIMError
from email import message_from_string
from email.utils import parseaddr
...@@ -13,6 +13,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -13,6 +13,7 @@ class EmailServiceTests(unittest.TestCase):
def setUp(self): def setUp(self):
self.settings = conftests.options.parse_settings() self.settings = conftests.options.parse_settings()
self.sm_client = conftests.sendmail.Sendmail(self.settings) self.sm_client = conftests.sendmail.Sendmail(self.settings)
self.locales = conftests.strings.get_locales()
def tearDown(self): def tearDown(self):
print("tearDown()") print("tearDown()")
...@@ -25,6 +26,38 @@ class EmailServiceTests(unittest.TestCase): ...@@ -25,6 +26,38 @@ class EmailServiceTests(unittest.TestCase):
request = ep.parse("From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org") request = ep.parse("From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org")
self.assertEqual(request["command"], "help") self.assertEqual(request["command"], "help")
def test_normalize_msg(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
msg_str = "From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org"
msg = conftests.message_from_string(msg_str)
request = ep.normalize(msg)
self.assertEqual(request, ('silvia [hiro]', 'hiro@torproject.org', '', 'gettor@torproject.org'))
def test_validate_msg(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
msg_str = "From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org"
msg = conftests.message_from_string(msg_str)
request = ep.validate("hiro@torproject.org", msg)
assert request
def test_dkim_verify(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
msg_str = "From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org"
msg = conftests.message_from_string(msg_str)
request = ep.dkim_verify(msg, "hiro@torproject.org")
assert request
def test_build_request(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
msg_str = "From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: \r\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org\r\n osx es"
msg = conftests.message_from_string(msg_str)
languages = [*self.locales.keys()]
platforms = self.settings.get('platforms')
request = ep.build_request(msg_str, "hiro@torproject.org", languages, platforms)
self.assertEqual(request["command"], "links")
self.assertEqual(request["platform"], "osx")
self.assertEqual(request["language"], "es")
def test_language_email_parser(self): def test_language_email_parser(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org") ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
request = ep.parse("From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: \r\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org\n osx en") request = ep.parse("From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: \r\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org\n osx en")
......
...@@ -18,9 +18,6 @@ class EmailServiceTests(unittest.TestCase): ...@@ -18,9 +18,6 @@ class EmailServiceTests(unittest.TestCase):
def tearDown(self): def tearDown(self):
print("tearDown()") print("tearDown()")
def test_get_available_locales(self):
self.assertEqual({"en": "English", "es": "Español", "pt": "Português Brasil"}, self.locales)
def test_load_en_strings(self): def test_load_en_strings(self):
conftests.strings.load_strings("en") conftests.strings.load_strings("en")
self.assertEqual(conftests.strings._("smtp_mirrors_subject"), "[GetTor] Mirrors") self.assertEqual(conftests.strings._("smtp_mirrors_subject"), "[GetTor] Mirrors")
...@@ -33,5 +30,9 @@ class EmailServiceTests(unittest.TestCase): ...@@ -33,5 +30,9 @@ class EmailServiceTests(unittest.TestCase):
conftests.strings.load_strings("es") conftests.strings.load_strings("es")
self.assertEqual(conftests.strings._("smtp_help_subject"), "[GetTor] Ayuda") self.assertEqual(conftests.strings._("smtp_help_subject"), "[GetTor] Ayuda")
def test_locale_supported(self):
self.assertEqual(self.locales['en']['language'], "English")
self.assertEqual(self.locales['es']['locale'], "es-ES")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment