Commit 8f31517c authored by Barkin Simsek's avatar Barkin Simsek 🐢
Browse files

Add UpdateFetchers and its tests

parent afaf24ef
Pipeline #7414 passed with stages
in 2 minutes and 41 seconds
......@@ -38,6 +38,7 @@ elif args.update:
logger.info("Intializing CAPTCHA Monitor in update mode")
schedule.every().day.do(cm.update_domains)
schedule.every().hour.do(cm.update_relays)
schedule.every().day.do(cm.update_fetchers)
# Run all scheduled jobs at the beginning
schedule.run_all()
......
......@@ -10,6 +10,7 @@ from captchamonitor.utils.exceptions import ConfigInitError, DatabaseInitError
from captchamonitor.core.update_relays import UpdateRelays
from captchamonitor.core.update_domains import UpdateDomains
from captchamonitor.utils.small_scripts import node_id
from captchamonitor.core.update_fetchers import UpdateFetchers
class CaptchaMonitor:
......@@ -98,6 +99,14 @@ class CaptchaMonitor:
UpdateRelays(config=self.__config, db_session=self.__db_session)
def update_fetchers(self) -> None:
"""
Updates the list of fetchers in the database
"""
self.__logger.info("Started updating list of fetchers")
UpdateFetchers(config=self.__config, db_session=self.__db_session)
def analyze(self) -> None:
"""
Analyses the data recorded in the database
......
import logging
from typing import Set
from sqlalchemy.orm import sessionmaker
from captchamonitor.utils.config import Config
from captchamonitor.utils.models import Fetcher
class UpdateFetchers:
"""
Discovers the list of Docker containers dedicated to web browsers and adds
them to the database
"""
def __init__(
self,
config: Config,
db_session: sessionmaker, # pylint: disable=R0801
) -> None:
"""
Initializes UpdateFetchers
:param config: The config class instance that contains global configuration values
:type config: Config
:param db_session: Database session used to connect to the database
:type db_session: sessionmaker
"""
# Private class attributes
self.__logger = logging.getLogger(__name__)
self.__config: Config = config
self.__db_session: sessionmaker = db_session
# Calls to the class methods
self.update()
def __discover_browser_containers(self) -> Set:
"""
Parses the config file to see which web browsers are available
:return: List of web browsers
:rtype: Set
"""
browsers = set()
for config in self.__config.keys():
if "docker" in config and "container" in config:
name = config.split("_")
browsers.add(f"{name[1]}_{name[2]}")
return browsers
def update(self) -> None:
"""
Automatically discovers the new fetchers that are using Docker containers
and adds them to the database
"""
self.__logger.info(
"Updating the fetchers list by discovering the available Docker containers"
)
browsers = self.__discover_browser_containers()
for browser in browsers:
for uses_tor in [True, False]:
# Check if there is a matching fetcher in the database
query = (
self.__db_session.query(Fetcher)
.filter(Fetcher.method == browser)
.filter(
Fetcher.uses_tor == uses_tor,
)
)
# Insert the fetcher if it doesn't already exist'
if query.count() == 0:
fetcher = Fetcher(
method=browser,
uses_tor=uses_tor,
version="0",
)
self.__db_session.add(fetcher)
# Save changes to the database
self.__db_session.commit()
import unittest
from captchamonitor.utils.config import Config
from captchamonitor.utils.models import Fetcher
from captchamonitor.utils.database import Database
from captchamonitor.core.update_fetchers import UpdateFetchers
class TestUpdateFetchers(unittest.TestCase):
def setUp(self):
self.config = Config()
self.database = Database(
self.config["db_host"],
self.config["db_port"],
self.config["db_name"],
self.config["db_user"],
self.config["db_password"],
)
self.db_session = self.database.session()
self.db_fetcher_query = self.db_session.query(Fetcher)
def tearDown(self):
self.db_session.close()
def test_discover_browser_containers(self):
update_fetchers = UpdateFetchers(config=self.config, db_session=self.db_session)
browsers = update_fetchers._UpdateFetchers__discover_browser_containers()
self.assertGreater(len(browsers), 0)
self.assertIn("tor_browser", browsers)
def test_update_fetchers(self):
# Make sure the table is empty
self.assertEqual(self.db_fetcher_query.count(), 0)
update_fetchers = UpdateFetchers(config=self.config, db_session=self.db_session)
# Check if fetchers were inserted
self.assertNotEqual(self.db_fetcher_query.count(), 0)
self.assertNotEqual(
self.db_fetcher_query.filter(Fetcher.method == "tor_browser").count(),
0,
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment