From 7eea744ea6f891acb4beda2c56270cc96949b671 Mon Sep 17 00:00:00 2001 From: Cecylia Bocovich Date: Fri, 5 Jun 2020 14:22:35 -0400 Subject: [PATCH 1/6] Refactor twitter service to match email service We made some improvements and fixed some bugs in the email service earlier. This commit updates the twitter service as well. --- gettor/services/twitter/twitterdm.py | 226 +++++++++++++-------------- 1 file changed, 113 insertions(+), 113 deletions(-) diff --git a/gettor/services/twitter/twitterdm.py b/gettor/services/twitter/twitterdm.py index 109f06d..ffcb863 100644 --- a/gettor/services/twitter/twitterdm.py +++ b/gettor/services/twitter/twitterdm.py @@ -98,6 +98,73 @@ class Twitterdm(object): return post_data + def build_locale_string(self, locales): + locale_string = "" + for locale in locales: + locale_string += "\t" + locale[0] + "\n" + return locale_string + + def build_help_body_message(self, locale_string): + body_msg = strings._("body_intro") + body_msg += strings._("help_body_intro") + body_msg += strings._("help_body_support") + body_msg += "\twindows\n\tlinux\n\tosx\n\n" + body_msg += strings._("help_body_respond") + body_msg += strings._("help_body_locale") + body_msg += locale_string + "\n" + body_msg += strings._("help_body_example").format("Windows", "Arabic", "windows ar") + + return body_msg + + + def build_link_strings(self, links, platform, locale): + """ + Build the links strings + """ + + link_msg = "" + + for link in links: + provider = link[5] + version = link[4] + arch = link[3] + url = link[0] + file = link[7] + sig_url = url + ".asc" + + link_str = "\t{}: {}\n".format(provider, url) + + link_str += "\tSignature file: {}\n".format(sig_url) + + link_msg = "{}\n{}".format(link_msg, link_str) + + return link_msg, file + + + def build_body_message(self, link_msg, platform, file): + signature_strings = { + "windows":"links_body_windows", + "linux":"links_body_linux", + "osx":"links_body_osx" + } + signature_cmds = { + "windows":"gpgv --keyring .\\tor.keyring Downloads\\{0}.asc Downloads\\{0}", + "linux":"gpgv --keyring ./tor.keyring ~/Downloads/{}{{.asc,}}", + "osx":"gpgv --keyring ./tor.keyring ~/Downloads/{}{{.asc,}}" + } + body_msg = strings._("body_intro") + body_msg += strings._("links_body_platform").format(platform) + body_msg += strings._("links_body_step1").format(link_msg) + body_msg += strings._("links_body_archive").format(file) + body_msg += strings._("links_body_internet_archive") + body_msg += strings._("links_body_google_drive") + body_msg += strings._("links_body_step2") + body_msg += strings._(signature_strings[platform]) + body_msg += strings._("links_body_all").format(signature_cmds[platform].format(file)) + body_msg += strings._("links_body_step3") + + return body_msg + @defer.inlineCallbacks def get_new(self): """ @@ -119,134 +186,67 @@ class Twitterdm(object): ).addCallback(tp.parse_callback).addErrback(tp.parse_errback) del tp - # Manage help and links messages separately - help_requests = yield self.conn.get_requests( - status="ONHOLD", command="help", service="twitter" - ) - - link_requests = yield self.conn.get_requests( - status="ONHOLD", command="links", service="twitter" + requests = yield self.conn.get_requests( + status="ONHOLD", service="twitter" ) - if help_requests: - strings.load_strings("en") - try: - log.debug("Got new help request.") - - for request in help_requests: - ids = json.loads("{}".format(request[0].replace("'", '"'))) - message_id = ids['id'] - twitter_id = ids['twitter_handle'] - date = request[5] - - hid = hashlib.sha256(twitter_id.encode('utf-8')) - log.debug( - "Sending help message to {}.".format( - hid.hexdigest() - ) - ) - - body_msg = strings._("help_body_intro") - body_msg += strings._("help_body_paragraph") - body_msg += strings._("help_body_support") - - yield self.twitterdm( - twitter_id=twitter_id, - message=body_msg - ) - - yield self.conn.update_stats( - command="help", platform='', language='en', - service="twitter" - ) - - yield self.conn.update_request( - id=request[0], hid=hid.hexdigest(), status="SENT", - service="twitter", date=date - ) + try: + log.debug("Got new help request.") - except RuntimeError as e: - log.error("Error sending twitter message: {}.".format(e)) + for request in requests: + ids = json.loads("{}".format(request[0].replace("'", '"'))) + message_id = ids['id'] + twitter_id = ids['twitter_handle'] + command = request[1] + platform = request[2] + language = request[3] + date = request[5] - elif link_requests: - try: - log.debug("Got new links request.") + if not language: + language = 'en' - for request in link_requests: - ids = json.loads("{}".format(request[0].replace("'", '"'))) - message_id = ids['id'] - twitter_id = ids['twitter_handle'] - date = request[5] - platform = request[2] - language = request[3] + body_msg ="" - if not language: - language = 'en' + if command == "help": - locales = strings.get_locales() + locales = yield self.conn.get_locales() + locale_string = self.build_locale_string(locales) - strings.load_strings(language) - locale = locales[language]['locale'] + # build message + body_msg = self.build_help_body_message(locale_string) - log.debug("Getting links for {}.".format(platform)) + elif command == "links": + log.debug("Getting links for {} {}.".format(platform, language)) links = yield self.conn.get_links( - platform=platform, language=locale, status="ACTIVE" + platform=platform, language=language, status="ACTIVE" ) # build message - link_msg = None - file = "" - - for link in links: - provider = link[5] - version = link[4] - arch = link[3] - url = link[0] - file = link[7] - sig_url = url + ".asc" - - link_str = "Tor Browser {} for {}-{}-{} ({}): {}\n".format( - version, platform, locale, arch, provider, url - ) - - link_str += "Signature file: {}\n".format(sig_url) - - if link_msg: - link_msg = "{}\n{}".format(link_msg, link_str) - else: - link_msg = link_str - - body_msg = strings._("links_body_platform").format(platform) - body_msg += trings._("links_body_links").format(link_msg) - body_msg += trings._("links_body_archive") - body_msg += trings._("links_body_internet_archive") - body_msg += trings._("links_body_google_drive") - body_msg += trings._("links_body_internet_archive").format(file) - body_msg += trings._("links_body_ending") - - hid = hashlib.sha256(twitter_id.encode('utf-8')) - log.debug( - "Sending links to {}.".format( - hid.hexdigest() - ) + link_msg, file = self.build_link_strings(links, platform, language) + body_msg = self.build_body_message(link_msg, platform, file) + else: + log.warn("Invalid gettor command {}.".format(command)) + yield self.conn.remove_request( + id=id, service="email", date=date ) - yield self.twitterdm( - twitter_id=twitter_id, - message=body_msg - ) + log.debug("Sending {} message.".format(request[1])) - yield self.conn.update_stats( - command="links", platform=platform, language=locale, - service="twitter" - ) - yield self.conn.update_request( - id=request[0], hid=hid.hexdigest(), status="SENT", - service="twitter", date=date - ) + yield self.twitterdm( + twitter_id=twitter_id, + message=body_msg + ) + + yield self.conn.update_stats( + command="help", platform='', language='en', + service="twitter" + ) + + yield self.conn.remove_request( + id=request[0], service="twitter", date=date + ) + + except RuntimeError as e: + log.error("Error sending twitter message: {}.".format(e)) - except RuntimeError as e: - log.error("Error sending message: {}.".format(e)) - else: - log.debug("No pending twitter requests. Keep waiting.") -- GitLab From f6f28c89dc8ea51630c43d9beae71573db52eea1 Mon Sep 17 00:00:00 2001 From: Cecylia Bocovich Date: Fri, 5 Jun 2020 15:09:29 -0400 Subject: [PATCH 2/6] Refactor parsers to share code Both the twitter and email parsers have to parse through a string looking for the same keywords. This refactors the parser objects to inherit from a parent class that contains generic parsing and request building functions. --- gettor/parse/email.py | 81 +++--------------------------------- gettor/parse/parser.py | 92 +++++++++++++++++++++++++++++++++++++++++ gettor/parse/twitter.py | 36 +++------------- tests/test_twitter.py | 2 +- 4 files changed, 103 insertions(+), 108 deletions(-) create mode 100644 gettor/parse/parser.py diff --git a/gettor/parse/email.py b/gettor/parse/email.py index fa945ba..f379809 100644 --- a/gettor/parse/email.py +++ b/gettor/parse/email.py @@ -12,8 +12,6 @@ from __future__ import absolute_import -import re -import io import dkim import hashlib @@ -29,6 +27,8 @@ from twisted.internet import defer from ..utils.db import SQLite3 from ..utils import validate_email +from gettor.parse.parser import Parser + class AddressError(Exception): """ Error if email address is not valid or it can't be normalized. @@ -43,7 +43,7 @@ class DKIMError(Exception): pass -class EmailParser(object): +class EmailParser(Parser): """Class for parsing email requests.""" def __init__(self, settings, to_addr=None, dkim=False): @@ -52,15 +52,9 @@ class EmailParser(object): param (Boolean) dkim: Set dkim verification to True or False. """ - self.settings = settings self.dkim = dkim self.to_addr = to_addr - self.locales = [] - self.platforms = self.settings.get("platforms") - self.conn = SQLite3(self.settings.get("dbname")) - - def __del__(self): - del self.conn + super().__init__(settings) def normalize(self, msg): # Normalization will convert alice@wonderland.net @@ -121,72 +115,6 @@ class EmailParser(object): else: return True - def parse_keywords(self, text, request): - - buf = io.StringIO(text) - for line in buf: - if len(line.strip()) > 0 and line.strip()[0] == ">": - continue - for word in re.split(r"\s+", line.strip()): - word = word.lower() - for locale in self.locales: - if word == locale.lower(): - request["language"] = locale - elif not request["language"]: - parts = re.split(r"[-_]", word) - if parts[0] == locale.lower()[:2]: - request["language"] = locale - if word in self.platforms: - request["command"] = "links" - request["platform"] = word - if (not request["command"]) and word == "help": - request["command"] = "help" - return request - - def build_request(self, msg_str, norm_addr): - # Search for commands keywords - subject_re = re.compile(r"Subject: (.*)\r\n") - subject = subject_re.search(msg_str) - - request = { - "id": norm_addr, - "command": None, - "platform": None, - "language": None, - "service": "email" - } - - if subject: - subject = subject.group(1) - request = self.parse_keywords(subject, request) - - # Always parse the body too, to see if there's more specific information - request = self.parse_keywords(msg_str, request) - - if not request["language"]: - request["language"] = "en-US" - - if not request["command"]: - request["command"] = "help" - - return request - - - def too_many_requests(self, hid, test_hid, num_requests, limit): - if hid == test_hid: - return False - elif num_requests < limit: - return False - else: - return True - - @defer.inlineCallbacks - def get_locales(self): - - locales = yield self.conn.get_locales() - for l in locales: - self.locales.append(l[0]) - def parse(self, msg_str): """ @@ -231,6 +159,7 @@ class EmailParser(object): log.msg("DKIM error: {}".format(e.args)) request = self.build_request(msg_str, norm_addr) + request["service"] = "email" return request diff --git a/gettor/parse/parser.py b/gettor/parse/parser.py new file mode 100644 index 0000000..09bb02f --- /dev/null +++ b/gettor/parse/parser.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor, a Tor Browser distribution system. +# +# :authors: Cecylia Bocovich +# see also AUTHORS file +# +# :copyright: (c) 2008-2020, The Tor Project, Inc. +# +# :license: This is Free Software. See LICENSE for license information. + +import re +import io + +from twisted.python import log +from twisted.internet import defer + +from ..utils.db import SQLite3 + +class Parser(object): + """Class for parsing gettor requests.""" + + def __init__(self, settings): + """ + Constructor. + + param (Boolean) dkim: Set dkim verification to True or False. + """ + self.settings = settings + self.locales = [] + self.platforms = self.settings.get("platforms") + self.conn = SQLite3(self.settings.get("dbname")) + + def __del__(self): + del self.conn + + def parse_keywords(self, text, request): + + buf = io.StringIO(text) + for line in buf: + if len(line.strip()) > 0 and line.strip()[0] == ">": + continue + for word in re.split(r"\s+", line.strip()): + word = word.lower() + for locale in self.locales: + if word == locale.lower(): + request["language"] = locale + elif not request["language"]: + parts = re.split(r"[-_]", word) + if parts[0] == locale.lower()[:2]: + request["language"] = locale + if word in self.platforms: + request["command"] = "links" + request["platform"] = word + if (not request["command"]) and word == "help": + request["command"] = "help" + return request + + def build_request(self, msg_str, sender): + # Search for commands keywords + request = { + "id": sender, + "command": None, + "platform": None, + "language": None + } + + request = self.parse_keywords(msg_str, request) + + if not request["language"]: + request["language"] = "en-US" + + if not request["command"]: + request["command"] = "help" + + return request + + def too_many_requests(self, hid, test_hid, num_requests, limit): + if hid == test_hid: + return False + elif num_requests < limit: + return False + else: + return True + + @defer.inlineCallbacks + def get_locales(self): + + locales = yield self.conn.get_locales() + for l in locales: + self.locales.append(l[0]) + diff --git a/gettor/parse/twitter.py b/gettor/parse/twitter.py index fd197ac..1352599 100644 --- a/gettor/parse/twitter.py +++ b/gettor/parse/twitter.py @@ -25,44 +25,17 @@ from twisted.internet import defer from ..utils.db import SQLite3 from ..utils import strings +from gettor.parse.parser import Parser -class TwitterParser(object): +class TwitterParser(Parser): """Class for parsing twitter message requests.""" def __init__(self, settings, twitter_id=None): """ Constructor. """ - self.settings = settings self.twitter_id = twitter_id - self.conn = SQLite3(self.settings.get("dbname")) - - def __del__(self): - del self.conn - - def build_request(self, msg_text, twitter_id, languages, platforms): - - request = { - "id": twitter_id, - "command": None, - "platform": None, - "language": "en", - "service": "twitter" - } - - if msg_text: - for word in re.split(r"\s+", msg_text.strip()): - if word.lower() in languages: - request["language"] = word.lower() - if word.lower() in platforms: - request["command"] = "links" - request["platform"] = word.lower() - if word.lower() == "help": - request["command"] = "help" - break - - return request - + super().__init__(settings) def parse(self, msg, twitter_id): """ @@ -87,7 +60,8 @@ class TwitterParser(object): "Request from {}".format(hid.hexdigest()), system="twitter parser" ) - request = self.build_request(msg, twitter_id, languages, platforms) + request = self.build_request(msg, twitter_id) + request["service"] = "twitter" return request diff --git a/tests/test_twitter.py b/tests/test_twitter.py index 1ad604d..c7210dc 100644 --- a/tests/test_twitter.py +++ b/tests/test_twitter.py @@ -24,7 +24,7 @@ class TwitterTests(unittest.TestCase): message = e['message_create']['message_data']['text'] tp = conftests.TwitterParser(self.settings, message_id) r = tp.parse(message, str(message_id)) - self.assertEqual(r, {'command': 'links', 'id': "{'id': '1178649287208689669', 'twitter_handle': '1467062174'}", 'language': 'en', 'platform': 'windows','service': 'twitter'}) + self.assertEqual(r, {'command': 'links', 'id': "{'id': '1178649287208689669', 'twitter_handle': '1467062174'}", 'language': 'en-US', 'platform': 'windows','service': 'twitter'}) if __name__ == "__main__": -- GitLab From c1c6a9c6cddec39bf3b38aca70de2cb90d015239 Mon Sep 17 00:00:00 2001 From: Cecylia Bocovich Date: Tue, 5 May 2020 15:31:57 -0400 Subject: [PATCH 3/6] Make sure file is defined We shouldn't crash the program if there are no links --- gettor/services/email/sendmail.py | 1 + gettor/services/twitter/twitterdm.py | 1 + 2 files changed, 2 insertions(+) diff --git a/gettor/services/email/sendmail.py b/gettor/services/email/sendmail.py index a877ede..1469894 100644 --- a/gettor/services/email/sendmail.py +++ b/gettor/services/email/sendmail.py @@ -117,6 +117,7 @@ class Sendmail(object): """ link_msg = "" + file = "" for link in links: provider = link[5] diff --git a/gettor/services/twitter/twitterdm.py b/gettor/services/twitter/twitterdm.py index ffcb863..5d2bd18 100644 --- a/gettor/services/twitter/twitterdm.py +++ b/gettor/services/twitter/twitterdm.py @@ -123,6 +123,7 @@ class Twitterdm(object): """ link_msg = "" + file = "" for link in links: provider = link[5] -- GitLab From 1c896179f65980fd71592fe6bbfe89e36e97068d Mon Sep 17 00:00:00 2001 From: Cecylia Bocovich Date: Tue, 5 May 2020 15:04:04 -0400 Subject: [PATCH 4/6] Fix typo in call to time.sleep() --- gettor/services/twitter/twitterdm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gettor/services/twitter/twitterdm.py b/gettor/services/twitter/twitterdm.py index 5d2bd18..be3fbf7 100644 --- a/gettor/services/twitter/twitterdm.py +++ b/gettor/services/twitter/twitterdm.py @@ -91,7 +91,7 @@ class Twitterdm(object): twitter_id, message ) if post_data.status_code == 200: - time.sleeps(61) + time.sleep(61) else: raise RuntimeError("Error sending message: (%s)" % str(e)) -- GitLab From 8aba408bc1120bbbd36304921dc534918845a5a4 Mon Sep 17 00:00:00 2001 From: Cecylia Bocovich Date: Tue, 5 May 2020 15:38:08 -0400 Subject: [PATCH 5/6] Catch and log all exceptions --- gettor/services/twitter/twitterdm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gettor/services/twitter/twitterdm.py b/gettor/services/twitter/twitterdm.py index be3fbf7..0e8989d 100644 --- a/gettor/services/twitter/twitterdm.py +++ b/gettor/services/twitter/twitterdm.py @@ -248,6 +248,6 @@ class Twitterdm(object): id=request[0], service="twitter", date=date ) - except RuntimeError as e: + except Exception as e: log.error("Error sending twitter message: {}.".format(e)) -- GitLab From 0fc62abd818d8dc8565dcc009b4f660b9aff404b Mon Sep 17 00:00:00 2001 From: Cecylia Bocovich Date: Sat, 6 Jun 2020 12:00:55 -0400 Subject: [PATCH 6/6] Remove callbacks from send tweet functions These functions don't take callbacks and errbacks. --- gettor/services/twitter/twitterdm.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/gettor/services/twitter/twitterdm.py b/gettor/services/twitter/twitterdm.py index 0e8989d..b45ed7a 100644 --- a/gettor/services/twitter/twitterdm.py +++ b/gettor/services/twitter/twitterdm.py @@ -53,23 +53,6 @@ class Twitterdm(object): return self.settings.get("twitter_interval") - def twitter_callback(self, message): - """ - Callback invoked after a message has been sent. - - :param message (string): Success details from the server. - """ - log.debug("Message sent successfully.") - - - def twitter_errback(self, error): - """ - Errback if we don't/can't send the message. - """ - log.warn("Could not send message.") - raise RuntimeError("{}".format(error)) - - def twitterdm(self, twitter_id, message): """ Send a twitter message for each message received. It creates a plain @@ -81,12 +64,6 @@ class Twitterdm(object): :return: deferred whose callback/errback will handle the API execution details. """ - return self.send_tweet( - twitter_id, message - ).addCallback(self.twitter_callback).addErrback(self.twitter_errback) - - - def send_tweet(self, twitter_id, message): post_data = self.twitter.post_message( twitter_id, message ) @@ -96,7 +73,6 @@ class Twitterdm(object): else: raise RuntimeError("Error sending message: (%s)" % str(e)) - return post_data def build_locale_string(self, locales): locale_string = "" @@ -234,7 +210,7 @@ class Twitterdm(object): log.debug("Sending {} message.".format(request[1])) - yield self.twitterdm( + self.twitterdm( twitter_id=twitter_id, message=body_msg ) -- GitLab