Commit 5fb69634 authored by Barkin Simsek's avatar Barkin Simsek 🐢
Browse files

Add relay updating functionality to main

parent 639c38ce
Pipeline #7211 passed with stages
in 2 minutes and 27 seconds
......@@ -8,10 +8,10 @@ build:
docker-compose build
up:
docker-compose up -d --scale cm-worker=5
docker-compose up -d --scale cm-worker=5 --scale cm-update=1
temp:
docker-compose up --scale cm-worker=5
docker-compose up --scale cm-worker=5 --scale cm-update=1
down:
ifneq ($(shell docker ps -f ancestor="captchamonitor-tor-container" -q),)
......@@ -21,11 +21,11 @@ endif
docker-compose down --remove-orphans
test: down
docker-compose up -d --scale cm-worker=0
docker-compose up -d --scale cm-worker=0 --scale cm-update=0
docker-compose run --rm --no-deps --entrypoint="pytest -v --reruns 3 --reruns-delay 3 --cov=/src/captchamonitor/ --cov-report term-missing" captchamonitor /tests
logs:
docker-compose logs --tail=100 captchamonitor cm-worker
docker-compose logs --tail=100 captchamonitor cm-worker cm-update
init: check_root
apt install python3-pip mypy
......
......@@ -38,6 +38,15 @@ services:
- captchamonitor
- --worker
cm-update:
<<: *captchamonitor_base_service
restart: always
entrypoint:
- python
- -m
- captchamonitor
- --update
postgres:
image: postgres:9.6
environment:
......
......@@ -6,5 +6,5 @@ disallow_untyped_calls = True
disallow_untyped_defs = True
disallow_incomplete_defs = True
[mypy-pytest.*,sqlalchemy.*,stem.*,docker.*,port_for.*,setuptools.*,sqlalchemy_utils.*,selenium.*,country_converter.*,bs4.*]
[mypy-pytest.*,sqlalchemy.*,stem.*,docker.*,port_for.*,setuptools.*,sqlalchemy_utils.*,selenium.*,country_converter.*,bs4.*,schedule.*]
ignore_missing_imports = True
\ No newline at end of file
# import time
# import schedule
import time
import logging
import argparse
import schedule
from captchamonitor.cm import CaptchaMonitor
parser = argparse.ArgumentParser(description="CAPTCHA Monitor")
......@@ -13,6 +14,13 @@ parser.add_argument(
default=False,
help="Run an instance of worker",
)
parser.add_argument(
"-u",
"--update",
action="store_true",
default=False,
help="Update the URLs and relays in the database",
)
args = parser.parse_args()
# Get the root logger for the package
......@@ -20,17 +28,21 @@ logging.basicConfig(format="%(asctime)s %(module)s [%(levelname)s] %(message)s")
logger = logging.getLogger("captchamonitor")
logger.setLevel(logging.DEBUG)
cm = CaptchaMonitor()
# Run in the specified mode
if args.worker:
logger.info("Intializing CAPTCHA Monitor in worker mode")
CaptchaMonitor().worker()
cm.worker()
elif args.update:
logger.info("Intializing CAPTCHA Monitor in update mode")
schedule.every().day.do(cm.update_urls)
schedule.every().hour.do(cm.update_relays)
# Schedule tasks
# logger.info("Scheduling tasks")
# schedule.every(0.5).seconds.do(cm.worker)
# Run all scheduled jobs at the beginning
schedule.run_all()
# # Run jobs
# logger.info("Started running the tasks")
# while True:
# schedule.run_pending()
# time.sleep(1)
# Run jobs at the scheduled times
while True:
schedule.run_pending()
time.sleep(1)
......@@ -7,6 +7,7 @@ from captchamonitor.core.worker import Worker
from captchamonitor.utils.config import Config
from captchamonitor.utils.database import Database
from captchamonitor.utils.exceptions import ConfigInitError, DatabaseInitError
from captchamonitor.core.update_relays import UpdateRelays
from captchamonitor.utils.small_scripts import node_id
......@@ -52,7 +53,7 @@ class CaptchaMonitor:
time.sleep(3)
# Check if database connection was made
if not hasattr(self, "__database"):
if not hasattr(self, f"_{self.__class__.__name__}__database"):
self.__logger.warning(
"Could not initialize CAPTCHA Monitor since I couldn't connect to the database, exitting"
)
......@@ -65,14 +66,14 @@ class CaptchaMonitor:
"""
Adds new jobs to the database
"""
self.__logger.debug("Adding new jobs")
self.__logger.info("Adding new jobs")
def worker(self) -> None:
"""
Fetches a job from the database and processes it using Tor Browser or
other specified browsers. Inserts the result back into the database.
"""
self.__logger.debug("Running worker %s", self.__node_id)
self.__logger.info("Running worker %s", self.__node_id)
Worker(
worker_id=self.__node_id,
......@@ -84,7 +85,15 @@ class CaptchaMonitor:
"""
Updates the list of URLs in the database
"""
self.__logger.debug("Started updating URLs")
self.__logger.info("Started updating URLs")
def update_relays(self) -> None:
"""
Updates the list of relays in the database
"""
self.__logger.info("Started updating relays")
UpdateRelays(config=self.__config, db_session=self.__db_session)
def analyze(self) -> None:
"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment