Add method to close dbpool and call it

We were leaving connections to the database open, which causes some
calls to hang. This adds destructors to the SQLite3 class and the
classes that use it, and refactors some code to make one database
connection per class object. This also makes sure that the destructors
for objects that use databases are actually called.

gettor/parse/email.py

@@ -56,6 +56,10 @@ class EmailParser(object):
         self.to_addr = to_addr
         self.locales = []
         self.platforms = self.settings.get("platforms")
+        self.conn = SQLite3(self.settings.get("dbname"))
+
+    def __del__(self):
+        del self.conn
 
     def normalize(self, msg):
         # Normalization will convert <Alice Wonderland> alice@wonderland.net
@@ -164,10 +168,8 @@ class EmailParser(object):
 
     @defer.inlineCallbacks
     def get_locales(self):
-        dbname = self.settings.get("dbname")
-        conn = SQLite3(dbname)
 
-        locales = yield conn.get_locales()
+        locales = yield self.conn.get_locales()
         for l in locales:
             self.locales.append(l[0])
 
@@ -235,7 +237,6 @@ class EmailParser(object):
         now_str = datetime.now().strftime("%Y%m%d%H%M%S")
         dbname = self.settings.get("dbname")
         test_hid = self.settings.get("test_hid")
-        conn = SQLite3(dbname)
 
         if request["command"]:
 
@@ -247,7 +248,7 @@ class EmailParser(object):
                 system="email parser"
             )
 
-            num_requests = yield conn.get_num_requests(
+            num_requests = yield self.conn.get_num_requests(
                 id=hid, service=request_service
             )
 
@@ -262,7 +263,7 @@ class EmailParser(object):
                     ), system="email parser"
                 )
             else:
-                conn.new_request(
+                self.conn.new_request(
                     id=request['id'],
                     command=request['command'],
                     platform=request['platform'],

gettor/parse/twitter.py

@@ -35,7 +35,10 @@ class TwitterParser(object):
         """
         self.settings = settings
         self.twitter_id = twitter_id
+        self.conn = SQLite3(self.settings.get("dbname"))
 
+    def __del__(self):
+        del self.conn
 
     def build_request(self, msg_text, twitter_id, languages, platforms):
 
@@ -110,16 +113,14 @@ class TwitterParser(object):
 
         if request["command"]:
             now_str = datetime.now().strftime("%Y%m%d%H%M%S")
-            dbname = self.settings.get("dbname")
-            conn = SQLite3(dbname)
 
             hid = hashlib.sha256(str(request['id']).encode('utf-8'))
             # check limits first
-            num_requests = yield conn.get_num_requests(
+            num_requests = yield self.conn.get_num_requests(
                 id=hid.hexdigest(), service=request['service']
             )
 
-            num_requests += yield conn.get_num_requests(
+            num_requests += yield self.conn.get_num_requests(
                 id=str(request['id']), service=request['service']
             )
 
@@ -130,7 +131,7 @@ class TwitterParser(object):
                     ), system="twitter parser"
                 )
             else:
-                conn.new_request(
+                self.conn.new_request(
                     id=str(request['id']),
                     command=request['command'],
                     platform=request['platform'],

gettor/services/email/sendmail.py

@@ -45,6 +45,8 @@ class Sendmail(object):
         dbname = self.settings.get("dbname")
         self.conn = DB(dbname)
 
+    def __del__(self):
+        del self.conn
 
     def get_interval(self):
         """

gettor/services/twitter/twitterdm.py

@@ -41,6 +41,8 @@ class Twitterdm(object):
         self.twitter = Twitter(settings)
         self.conn = DB(dbname)
 
+    def __del__(self):
+        del self.conn
 
     def get_interval(self):
         """
@@ -115,6 +117,7 @@ class Twitterdm(object):
             yield defer.maybeDeferred(
                 tp.parse, e['message_create']['message_data']['text'], message_id
             ).addCallback(tp.parse_callback).addErrback(tp.parse_errback)
+            del tp
 
         # Manage help and links messages separately
         help_requests = yield self.conn.get_requests(

gettor/utils/db.py

@@ -24,6 +24,9 @@ class SQLite3(object):
 			"sqlite3", dbname, check_same_thread=False
 		)
 
+	def __del__(self):
+		self.dbpool.close()
+
 	def query_callback(self, results=None):
 		"""
 		Query callback

scripts/process_email

@@ -41,6 +41,7 @@ def process_email(message):
             log.err("DKIM error: {}".format(e), system="process email")
             reactor.stop()
 
+    del ep
     reactor.stop()
 
 def main():

tests/test_db.py

@@ -2,8 +2,6 @@
 import pytest
 import pytest_twisted
 from twisted.trial import unittest
-from twisted.internet import defer, reactor
-from twisted.internet import task
 
 from . import conftests
 
@@ -19,7 +17,7 @@ class DatabaseTests(unittest.TestCase):
 
     def tearDown(self):
         print("tearDown()")
-        return self.conn.dbpool.close()
+        del self.conn
 
     @pytest_twisted.inlineCallbacks
     def test_stored_locales(self):

tests/test_email_service.py

 #!/usr/bin/env python3
 import pytest
+import pytest_twisted
 import hashlib
 from datetime import datetime
 from twisted.trial import unittest
@@ -31,6 +32,7 @@ class EmailServiceTests(unittest.TestCase):
 
     def tearDown(self):
         print("tearDown()")
+        del self.sm_client
 
     def test_get_interval(self):
         self.assertEqual(self.settings.get("sendmail_interval"), self.sm_client.get_interval())
@@ -39,6 +41,7 @@ class EmailServiceTests(unittest.TestCase):
         ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
         request = ep.parse("From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org")
         self.assertEqual(request["command"], "help")
+        del ep
 
     def test_normalize_msg(self):
         ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@@ -46,6 +49,7 @@ class EmailServiceTests(unittest.TestCase):
         msg = conftests.message_from_string(msg_str)
         request = ep.normalize(msg)
         self.assertEqual(request, ('silvia [hiro]', 'hiro@torproject.org', '', 'gettor@torproject.org'))
+        del ep
 
     def test_validate_msg(self):
         ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@@ -53,6 +57,7 @@ class EmailServiceTests(unittest.TestCase):
         msg = conftests.message_from_string(msg_str)
         request = ep.validate("hiro@torproject.org", msg)
         assert request
+        del ep
 
     def test_dkim_verify(self):
         ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@@ -60,6 +65,7 @@ class EmailServiceTests(unittest.TestCase):
         msg = conftests.message_from_string(msg_str)
         request = ep.dkim_verify(msg, "hiro@torproject.org")
         assert request
+        del ep
 
     def test_build_request(self):
         ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@@ -70,6 +76,7 @@ class EmailServiceTests(unittest.TestCase):
         self.assertEqual(request["command"], "links")
         self.assertEqual(request["platform"], "osx")
         self.assertEqual(request["language"], "es")
+        del ep
 
     def test_too_many_request_exclude(self):
         ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@@ -79,6 +86,7 @@ class EmailServiceTests(unittest.TestCase):
         check = ep.too_many_requests(hid, self.settings.get("test_hid"), num_requests, limit)
         self.assertEqual(hid, self.settings.get("test_hid"))
         self.assertEqual(check, False)
+        del ep
 
     def test_language_email_parser(self):
         ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@@ -137,6 +145,7 @@ class EmailServiceTests(unittest.TestCase):
                 "Subject: linux es-AR\r\n Reply-To: hiro@torproject.org \nTo:"
                 "gettor@torproject.org\n linux es")
         self.assertEqual(request["language"], "es-AR")
+        del ep
 
     def test_sent_links_message(self):
         ep = self.sm_client
@@ -159,6 +168,13 @@ class EmailServiceTests(unittest.TestCase):
         help_msg = ep.build_help_body_message()
         assert "This is how you can request a tor browser bundle link" in help_msg
 
+    @pytest_twisted.inlineCallbacks
+    def test_get_locales(self):
+        ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
+        yield ep.get_locales().addErrback(ep.parse_errback)
+        assert "en-US" in ep.locales
+        del ep
+
 
 if __name__ == "__main__":
     unittest.main()