GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Commit c413a8eb authored by Cecylia Bocovich's avatar Cecylia Bocovich

Add method to close dbpool and call it

We were leaving connections to the database open, which causes some
calls to hang. This adds destructors to the SQLite3 class and the
classes that use it, and refactors some code to make one database
connection per class object. This also makes sure that the destructors
for objects that use databases are actually called.
parent c6cfd912
...@@ -56,6 +56,10 @@ class EmailParser(object): ...@@ -56,6 +56,10 @@ class EmailParser(object):
self.to_addr = to_addr self.to_addr = to_addr
self.locales = [] self.locales = []
self.platforms = self.settings.get("platforms") self.platforms = self.settings.get("platforms")
self.conn = SQLite3(self.settings.get("dbname"))
def __del__(self):
del self.conn
def normalize(self, msg): def normalize(self, msg):
# Normalization will convert <Alice Wonderland> alice@wonderland.net # Normalization will convert <Alice Wonderland> alice@wonderland.net
...@@ -164,10 +168,8 @@ class EmailParser(object): ...@@ -164,10 +168,8 @@ class EmailParser(object):
@defer.inlineCallbacks @defer.inlineCallbacks
def get_locales(self): def get_locales(self):
dbname = self.settings.get("dbname")
conn = SQLite3(dbname)
locales = yield conn.get_locales() locales = yield self.conn.get_locales()
for l in locales: for l in locales:
self.locales.append(l[0]) self.locales.append(l[0])
...@@ -235,7 +237,6 @@ class EmailParser(object): ...@@ -235,7 +237,6 @@ class EmailParser(object):
now_str = datetime.now().strftime("%Y%m%d%H%M%S") now_str = datetime.now().strftime("%Y%m%d%H%M%S")
dbname = self.settings.get("dbname") dbname = self.settings.get("dbname")
test_hid = self.settings.get("test_hid") test_hid = self.settings.get("test_hid")
conn = SQLite3(dbname)
if request["command"]: if request["command"]:
...@@ -247,7 +248,7 @@ class EmailParser(object): ...@@ -247,7 +248,7 @@ class EmailParser(object):
system="email parser" system="email parser"
) )
num_requests = yield conn.get_num_requests( num_requests = yield self.conn.get_num_requests(
id=hid, service=request_service id=hid, service=request_service
) )
...@@ -262,7 +263,7 @@ class EmailParser(object): ...@@ -262,7 +263,7 @@ class EmailParser(object):
), system="email parser" ), system="email parser"
) )
else: else:
conn.new_request( self.conn.new_request(
id=request['id'], id=request['id'],
command=request['command'], command=request['command'],
platform=request['platform'], platform=request['platform'],
......
...@@ -35,7 +35,10 @@ class TwitterParser(object): ...@@ -35,7 +35,10 @@ class TwitterParser(object):
""" """
self.settings = settings self.settings = settings
self.twitter_id = twitter_id self.twitter_id = twitter_id
self.conn = SQLite3(self.settings.get("dbname"))
def __del__(self):
del self.conn
def build_request(self, msg_text, twitter_id, languages, platforms): def build_request(self, msg_text, twitter_id, languages, platforms):
...@@ -110,16 +113,14 @@ class TwitterParser(object): ...@@ -110,16 +113,14 @@ class TwitterParser(object):
if request["command"]: if request["command"]:
now_str = datetime.now().strftime("%Y%m%d%H%M%S") now_str = datetime.now().strftime("%Y%m%d%H%M%S")
dbname = self.settings.get("dbname")
conn = SQLite3(dbname)
hid = hashlib.sha256(str(request['id']).encode('utf-8')) hid = hashlib.sha256(str(request['id']).encode('utf-8'))
# check limits first # check limits first
num_requests = yield conn.get_num_requests( num_requests = yield self.conn.get_num_requests(
id=hid.hexdigest(), service=request['service'] id=hid.hexdigest(), service=request['service']
) )
num_requests += yield conn.get_num_requests( num_requests += yield self.conn.get_num_requests(
id=str(request['id']), service=request['service'] id=str(request['id']), service=request['service']
) )
...@@ -130,7 +131,7 @@ class TwitterParser(object): ...@@ -130,7 +131,7 @@ class TwitterParser(object):
), system="twitter parser" ), system="twitter parser"
) )
else: else:
conn.new_request( self.conn.new_request(
id=str(request['id']), id=str(request['id']),
command=request['command'], command=request['command'],
platform=request['platform'], platform=request['platform'],
......
...@@ -45,6 +45,8 @@ class Sendmail(object): ...@@ -45,6 +45,8 @@ class Sendmail(object):
dbname = self.settings.get("dbname") dbname = self.settings.get("dbname")
self.conn = DB(dbname) self.conn = DB(dbname)
def __del__(self):
del self.conn
def get_interval(self): def get_interval(self):
""" """
......
...@@ -41,6 +41,8 @@ class Twitterdm(object): ...@@ -41,6 +41,8 @@ class Twitterdm(object):
self.twitter = Twitter(settings) self.twitter = Twitter(settings)
self.conn = DB(dbname) self.conn = DB(dbname)
def __del__(self):
del self.conn
def get_interval(self): def get_interval(self):
""" """
...@@ -115,6 +117,7 @@ class Twitterdm(object): ...@@ -115,6 +117,7 @@ class Twitterdm(object):
yield defer.maybeDeferred( yield defer.maybeDeferred(
tp.parse, e['message_create']['message_data']['text'], message_id tp.parse, e['message_create']['message_data']['text'], message_id
).addCallback(tp.parse_callback).addErrback(tp.parse_errback) ).addCallback(tp.parse_callback).addErrback(tp.parse_errback)
del tp
# Manage help and links messages separately # Manage help and links messages separately
help_requests = yield self.conn.get_requests( help_requests = yield self.conn.get_requests(
......
...@@ -24,6 +24,9 @@ class SQLite3(object): ...@@ -24,6 +24,9 @@ class SQLite3(object):
"sqlite3", dbname, check_same_thread=False "sqlite3", dbname, check_same_thread=False
) )
def __del__(self):
self.dbpool.close()
def query_callback(self, results=None): def query_callback(self, results=None):
""" """
Query callback Query callback
......
...@@ -41,6 +41,7 @@ def process_email(message): ...@@ -41,6 +41,7 @@ def process_email(message):
log.err("DKIM error: {}".format(e), system="process email") log.err("DKIM error: {}".format(e), system="process email")
reactor.stop() reactor.stop()
del ep
reactor.stop() reactor.stop()
def main(): def main():
......
...@@ -2,8 +2,6 @@ ...@@ -2,8 +2,6 @@
import pytest import pytest
import pytest_twisted import pytest_twisted
from twisted.trial import unittest from twisted.trial import unittest
from twisted.internet import defer, reactor
from twisted.internet import task
from . import conftests from . import conftests
...@@ -19,7 +17,7 @@ class DatabaseTests(unittest.TestCase): ...@@ -19,7 +17,7 @@ class DatabaseTests(unittest.TestCase):
def tearDown(self): def tearDown(self):
print("tearDown()") print("tearDown()")
return self.conn.dbpool.close() del self.conn
@pytest_twisted.inlineCallbacks @pytest_twisted.inlineCallbacks
def test_stored_locales(self): def test_stored_locales(self):
......
#!/usr/bin/env python3 #!/usr/bin/env python3
import pytest import pytest
import pytest_twisted
import hashlib import hashlib
from datetime import datetime from datetime import datetime
from twisted.trial import unittest from twisted.trial import unittest
...@@ -31,6 +32,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -31,6 +32,7 @@ class EmailServiceTests(unittest.TestCase):
def tearDown(self): def tearDown(self):
print("tearDown()") print("tearDown()")
del self.sm_client
def test_get_interval(self): def test_get_interval(self):
self.assertEqual(self.settings.get("sendmail_interval"), self.sm_client.get_interval()) self.assertEqual(self.settings.get("sendmail_interval"), self.sm_client.get_interval())
...@@ -39,6 +41,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -39,6 +41,7 @@ class EmailServiceTests(unittest.TestCase):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org") ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
request = ep.parse("From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org") request = ep.parse("From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org")
self.assertEqual(request["command"], "help") self.assertEqual(request["command"], "help")
del ep
def test_normalize_msg(self): def test_normalize_msg(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org") ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
...@@ -46,6 +49,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -46,6 +49,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str) msg = conftests.message_from_string(msg_str)
request = ep.normalize(msg) request = ep.normalize(msg)
self.assertEqual(request, ('silvia [hiro]', 'hiro@torproject.org', '', 'gettor@torproject.org')) self.assertEqual(request, ('silvia [hiro]', 'hiro@torproject.org', '', 'gettor@torproject.org'))
del ep
def test_validate_msg(self): def test_validate_msg(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org") ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
...@@ -53,6 +57,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -53,6 +57,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str) msg = conftests.message_from_string(msg_str)
request = ep.validate("hiro@torproject.org", msg) request = ep.validate("hiro@torproject.org", msg)
assert request assert request
del ep
def test_dkim_verify(self): def test_dkim_verify(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org") ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
...@@ -60,6 +65,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -60,6 +65,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str) msg = conftests.message_from_string(msg_str)
request = ep.dkim_verify(msg, "hiro@torproject.org") request = ep.dkim_verify(msg, "hiro@torproject.org")
assert request assert request
del ep
def test_build_request(self): def test_build_request(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org") ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
...@@ -70,6 +76,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -70,6 +76,7 @@ class EmailServiceTests(unittest.TestCase):
self.assertEqual(request["command"], "links") self.assertEqual(request["command"], "links")
self.assertEqual(request["platform"], "osx") self.assertEqual(request["platform"], "osx")
self.assertEqual(request["language"], "es") self.assertEqual(request["language"], "es")
del ep
def test_too_many_request_exclude(self): def test_too_many_request_exclude(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org") ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
...@@ -79,6 +86,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -79,6 +86,7 @@ class EmailServiceTests(unittest.TestCase):
check = ep.too_many_requests(hid, self.settings.get("test_hid"), num_requests, limit) check = ep.too_many_requests(hid, self.settings.get("test_hid"), num_requests, limit)
self.assertEqual(hid, self.settings.get("test_hid")) self.assertEqual(hid, self.settings.get("test_hid"))
self.assertEqual(check, False) self.assertEqual(check, False)
del ep
def test_language_email_parser(self): def test_language_email_parser(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org") ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
...@@ -137,6 +145,7 @@ class EmailServiceTests(unittest.TestCase): ...@@ -137,6 +145,7 @@ class EmailServiceTests(unittest.TestCase):
"Subject: linux es-AR\r\n Reply-To: hiro@torproject.org \nTo:" "Subject: linux es-AR\r\n Reply-To: hiro@torproject.org \nTo:"
"gettor@torproject.org\n linux es") "gettor@torproject.org\n linux es")
self.assertEqual(request["language"], "es-AR") self.assertEqual(request["language"], "es-AR")
del ep
def test_sent_links_message(self): def test_sent_links_message(self):
ep = self.sm_client ep = self.sm_client
...@@ -159,6 +168,13 @@ class EmailServiceTests(unittest.TestCase): ...@@ -159,6 +168,13 @@ class EmailServiceTests(unittest.TestCase):
help_msg = ep.build_help_body_message() help_msg = ep.build_help_body_message()
assert "This is how you can request a tor browser bundle link" in help_msg assert "This is how you can request a tor browser bundle link" in help_msg
@pytest_twisted.inlineCallbacks
def test_get_locales(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
yield ep.get_locales().addErrback(ep.parse_errback)
assert "en-US" in ep.locales
del ep
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment