Switch from twisted adbapi to sqlite3 package (e4f6d98f) · Commits · The Tor Project / Anti-censorship / GetTor Project / gettor

README.md

+2 −0

Original line number	Diff line number	Diff line
		@@ -82,5 +82,7 @@ GetTor includes PyTest unit tests. To run the tests, first install the dependenc


		```
		$ python3 scripts/create_db -n -c -o -f tests/gettor.db
		$ python3 scripts/add_links_to_db -f tests/gettor.db
		$ pytest-3 tests/
		```

gettor/utils/db.py

+41 −45

Original line number	Diff line number	Diff line
		@@ -9,10 +9,10 @@

		from __future__ import absolute_import

		import sqlite3
		from datetime import datetime

		from twisted.python import log
		from twisted.enterprise import adbapi

		class SQLite3(object):
		"""
		@@ -20,94 +20,90 @@ class SQLite3(object):
		"""
		def __init__(self, dbname):
		"""Constructor."""
		self.dbpool = adbapi.ConnectionPool(
		"sqlite3", dbname, check_same_thread=False
		)

		def query_callback(self, results=None):
		"""
		Query callback
		Log that the database query has been executed and return results
		"""
		log.msg("Database query executed successfully.")
		return results

		def query_errback(self, error=None):
		"""
		Query error callback
		Logs database error
		"""
		if error:
		log.msg("Database error: {}".format(error))
		return None
		self.conn = sqlite3.connect(dbname)

		def new_request(self, id, command, service, platform, language, date, status):
		"""
		Perform a new request to the database
		"""
		c = self.conn.cursor()
		query = "INSERT INTO requests VALUES(?, ?, ?, ?, ?, ?, ?)"

		return self.dbpool.runQuery(
		query, (id, command, platform, language, service, date, status)
		).addCallback(self.query_callback).addErrback(self.query_errback)
		c.execute(query, (id, command, platform, language, service,
		date, status))
		self.conn.commit()
		return

		def get_requests(self, status, command, service):
		"""
		Perform a SELECT request to the database
		"""
		c = self.conn.cursor()
		query = "SELECT * FROM requests WHERE service=? AND command=? AND "\
		"status = ?"

		return self.dbpool.runQuery(
		query, (service, command, status)
		).addCallback(self.query_callback).addErrback(self.query_errback)
		c.execute(query, (service, command, status))

		return c.fetchall()

		def get_num_requests(self, id, service):
		"""
		Get number of requests for statistics
		"""
		query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND service=?"
		c = self.conn.cursor()
		query = "SELECT COUNT(rowid) FROM requests WHERE id=? AND "\
		"service=?"

		return self.dbpool.runQuery(
		query, (id, service)
		).addCallback(self.query_callback).addErrback(self.query_errback)
		c.execute(query, (id, service))
		return c.fetchone()[0]

		def remove_request(self, id, service, date):
		"""
		Removes completed request record from the database
		"""
		query = "DELETE FROM requests WHERE id=? AND service=? AND date=?"
		c = self.conn.cursor()
		query = "DELETE FROM requests WHERE id=? AND service=? AND "\
		"date=?"

		return self.dbpool.runQuery(
		query, (id, service, date)
		).addCallback(self.query_callback).addErrback(self.query_errback)
		c.execute(query, (id, service, date))
		self.conn.commit()
		return

		def update_stats(self, command, service, platform=None, language='en'):
		"""
		Update statistics to the database
		"""
		c = self.conn.cursor()
		now_str = datetime.now().strftime("%Y%m%d")
		query = "INSERT INTO stats(num_requests, platform, language, command, "\
		"service, date) VALUES (1, ?, ?, ?, ?, ?) ON CONFLICT(platform, "\
		"language, command, service, date) DO UPDATE SET num_requests=num_requests+1"
		"service, date) VALUES (1, ?, ?, ?, ?, ?) ON "\
		"CONFLICT(platform, language, command, service, date) "\
		"DO UPDATE SET num_requests=num_requests+1"

		return self.dbpool.runQuery(
		query, (platform, language, command, service, now_str)
		).addCallback(self.query_callback).addErrback(self.query_errback)
		c.execute(query, (platform, language, command, service,
		now_str))
		self.conn.commit()
		return

		def get_links(self, platform, language, status):
		"""
		Get links from the database per platform
		"""
		c = self.conn.cursor()
		query = "SELECT * FROM links WHERE platform=? AND language=? AND status=?"
		return self.dbpool.runQuery(
		query, (platform, language, status)
		).addCallback(self.query_callback).addErrback(self.query_errback)
		c.execute(query, (platform, language, status))

		return c.fetchall()

		def get_locales(self):
		"""
		Get a list of the supported tor browser binary locales
		"""
		c = self.conn.cursor()
		query = "SELECT DISTINCT language FROM links"
		return self.dbpool.runQuery(query
		).addCallback(self.query_callback).addErrback(self.query_errback)
		c.execute(query)

		locales = []
		for locale in c.fetchall():
		locales.append(locale[0])
		return locales

tests/conftests.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@ from __future__ import unicode_literals
		from gettor.utils import options
		from gettor.utils import strings
		from gettor.utils import twitter
		from gettor.utils.db import SQLite3
		from gettor.services.email.sendmail import Sendmail
		from gettor.services.twitter import twitterdm
		from gettor.parse.email import EmailParser, AddressError, DKIMError