Commit 87368e5c authored by Barkin Simsek's avatar Barkin Simsek 🐢
Browse files

Add skeleton code and services for the dashboard

parent 72a79062
Pipeline #9882 passed with stages
in 3 minutes and 9 seconds
......@@ -22,4 +22,6 @@ CM_ASSET_HAR_EXPORT_EXTENSION_XPI=/src/captchamonitor/assets/har_export_trigger-
CM_ASSET_HAR_EXPORT_EXTENSION_XPI_ID=harexporttrigger@getfirebug.com
CM_ASSET_HAR_EXPORT_EXTENSION_CRX=/src/captchamonitor/assets/har_export_trigger-0.6.1.crx
CM_JOB_QUEUE_DELAY=1
CM_FIXTURE_LOCATION=/src/captchamonitor/fixtures
\ No newline at end of file
CM_FIXTURE_LOCATION=/src/captchamonitor/fixtures
CM_DASHBOARD_LOCATION=/src/captchamonitor/dashboard
CM_DASHBOARD_WWW_LOCATION=/src/captchamonitor/dashboard/www
\ No newline at end of file
......@@ -10,7 +10,7 @@ build:
up:
@echo "\e[93m>> Running all of the containers\e[0m"
docker-compose up -d --scale cm-worker=3 --scale cm-updater=1 --scale cm-analyzer=1
docker-compose up -d --scale cm-worker=3 --scale cm-updater=1 --scale cm-analyzer=1 --scale cm-dashboard=1
down:
@echo "\e[93m>> Shutting down the containers\e[0m"
......@@ -26,7 +26,7 @@ test: pretest
pretest: down
@echo "\e[93m>> Preparing the containers for testing\e[0m"
docker-compose up -d --scale cm-worker=0 --scale cm-updater=0 --scale cm-analyzer=0
docker-compose up -d --scale cm-worker=0 --scale cm-updater=0 --scale cm-analyzer=0 --scale cm-dashboard=0
singletest:
ifndef TEST
......@@ -39,7 +39,7 @@ endif
logs:
@echo "\e[93m>> Printing the logs\e[0m"
docker-compose logs --tail=100 captchamonitor cm-worker cm-updater cm-analyzer
docker-compose logs --tail=100 captchamonitor cm-worker cm-updater cm-analyzer cm-dashboard
init: check_root
@echo "\e[93m>> Creating .env file\e[0m"
......
......@@ -17,6 +17,8 @@ services:
depends_on:
postgres:
condition: service_healthy
nginx:
condition: service_healthy
tor-browser-container:
condition: service_healthy
firefox-browser-container:
......@@ -54,7 +56,7 @@ services:
- -m
- captchamonitor
- --analyzer
cm-updater:
<<: *captchamonitor_base_service
restart: always
......@@ -64,6 +66,15 @@ services:
- captchamonitor
- --updater
cm-dashboard:
<<: *captchamonitor_base_service
restart: always
entrypoint:
- python
- -m
- captchamonitor
- --dashboard
postgres:
<<: *default-logging-options
image: postgres:9.6
......@@ -82,6 +93,19 @@ services:
timeout: 5s
retries: 5
nginx:
<<: *default-logging-options
image: nginx
ports:
- "5000:80"
volumes:
- ./src/captchamonitor/dashboard/www:/usr/share/nginx/html:ro
healthcheck:
test: ["CMD", "service", "nginx", "status"]
interval: 60s
timeout: 30s
retries: 5
tor-container:
<<: *default-logging-options
image: captchamonitor-tor-container:latest
......@@ -92,6 +116,11 @@ services:
# Basically do nothing on purpose, this container will be created and run
# as needed by the main code
- /bin/sh
healthcheck:
test: ["CMD", "service", "tor", "status"]
interval: 60s
timeout: 30s
retries: 5
tor-browser-container:
<<: *default-logging-options
......@@ -181,7 +210,7 @@ services:
interval: 15s
timeout: 30s
retries: 5
networks:
captchamonitor_network:
driver: bridge
......
......@@ -29,6 +29,13 @@ parser.add_argument(
default=False,
help="Update the URLs and relays in the database",
)
parser.add_argument(
"-d",
"--dashboard",
action="store_true",
default=False,
help="Update the static dashboard code",
)
args = parser.parse_args()
# Get the root logger for the package
......@@ -55,6 +62,9 @@ elif args.updater:
schedule.every().hour.do(cm.update_proxies)
schedule.every().day.do(cm.update_fetchers)
schedule.every().hour.do(cm.schedule_jobs)
elif args.dashboard:
logger.info("Intializing CAPTCHA Monitor in dashboard update mode")
schedule.every(30).minutes.do(cm.render_dashboard)
# Run all scheduled jobs at the beginning
schedule.run_all()
......
......@@ -17,6 +17,7 @@ from captchamonitor.core.update_domains import UpdateDomains
from captchamonitor.core.update_proxies import UpdateProxies
from captchamonitor.utils.small_scripts import node_id, hasattr_private, insert_fixtures
from captchamonitor.core.update_fetchers import UpdateFetchers
from captchamonitor.dashboard.render_dashboard import RenderDashboard
class CaptchaMonitor:
......@@ -109,6 +110,16 @@ class CaptchaMonitor:
UpdateProxies(config=self.__config, db_session=self.__db_session)
def render_dashboard(self) -> None:
"""
Renders the dashboard HTML code again
"""
self.__logger.info("Rendering the dashboard")
RenderDashboard(config=self.__config, db_session=self.__db_session)
self.__logger.info("Done with rendering the dashboard")
def schedule_jobs(self) -> None:
"""
Adds new jobs to the database
......
import os
import shutil
import logging
from jinja2 import Environment, FileSystemLoader
from sqlalchemy.orm import sessionmaker
from captchamonitor.utils.config import Config
from captchamonitor.utils.models import Domain
class RenderDashboard:
"""
Renders the latest version of the dashboard and exports the HTML files to
www folder
"""
def __init__(self, config: Config, db_session: sessionmaker) -> None:
"""
Initializes the dashboard renderer
:param config: The config class instance that contains global configuration values
:type config: Config
:param db_session: Database session used to connect to the database
:type db_session: sessionmaker
"""
# Private class attributes
self.__logger = logging.getLogger(__name__)
self.__config: Config = config
self.__db_session: sessionmaker = db_session
self.__template_location: str = os.path.join(
self.__config["dashboard_location"], "templates"
)
self.__jinja_environment = Environment(
loader=FileSystemLoader(self.__template_location)
)
# Clean the www directory
self.cleanup_www_folder()
# Render all pages
self.render_index()
self.render_domain_list()
def __write_to_file(self, filename: str, html_data: str) -> None:
"""
Writes given data to the specified file inside the www folder
:param filename: Name of the HTML file to export
:type filename: str
:param html_data: HTML data to write to the file
:type html_data: str
"""
html_file_path = os.path.join(self.__config["dashboard_www_location"], filename)
with open(html_file_path, "w") as file:
file.write(html_data)
def cleanup_www_folder(self) -> None:
"""
Remove contents of the www folder and copy the static folder again
"""
self.__logger.debug("Cleaning up the www folder")
www_location = self.__config["dashboard_www_location"]
static_location = os.path.join(self.__config["dashboard_location"], "static")
# Remove folders and files inside the www folder
for filename in os.listdir(www_location):
file_path = os.path.join(www_location, filename)
if (os.path.isfile(file_path) or os.path.islink(file_path)) and (
filename != ".gitignore"
):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
# Copy static files
shutil.copytree(static_location, www_location, dirs_exist_ok=True)
def render_index(self, filename: str = "index.html") -> None:
"""
Render the index page, which visitors first see
:param filename: Name of the HTML file to export, defaults to "index.html"
:type filename: str
"""
self.__logger.debug("Rendering %s", filename)
template = self.__jinja_environment.get_template("index.html")
html_data = template.render()
self.__write_to_file(filename=filename, html_data=html_data)
def render_domain_list(self, filename: str = "domain_list.html") -> None:
"""
Render the page that contains the list of tracked domains
:param filename: Name of the HTML file to export, defaults to "domain_list.html"
:type filename: str
"""
self.__logger.debug("Rendering %s", filename)
data = self.__db_session.query(Domain).all()
template = self.__jinja_environment.get_template("domain_list.html")
html_data = template.render(data=data)
self.__write_to_file(filename=filename, html_data=html_data)
This diff is collapsed.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="shortcut icon" href="images/favicon.ico">
<link rel="stylesheet" href="css/bootstrap.min.css">
<title>{% block title %}{% endblock %}</title>
</head>
<body>
<div class="container">
{% block body%}
{% endblock %}
</div>
</body>
</html>
\ No newline at end of file
{% extends 'base.html' %}
{% block title%}Domain List{% endblock %}
{% block body %}
<h1>Domain List</h1>
<table class="table">
<tr>
<th scope="col">Domain Name</th>
<th scope="col">HTTP Support</th>
<th scope="col">HTTPS Support</th>
<th scope="col">FTP Support</th>
<th scope="col">IPv4 Support</th>
<th scope="col">IPv6 Support</th>
</tr>
<tbody>
{% for domain in data %}
<tr>
<td scope="row">{{domain.domain}}</td>
<td>{{domain.supports_http}}</td>
<td>{{domain.supports_https}}</td>
<td>{{domain.supports_ftp}}</td>
<td>{{domain.supports_ipv4}}</td>
<td>{{domain.supports_ipv6}}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}
\ No newline at end of file
{% extends 'base.html' %}
{% block title%}CAPTCHA Monitor{% endblock %}
{% block body %}
<a href="domain_list.html">Domain List</a>
{% endblock %}
\ No newline at end of file
*
!.gitignore
\ No newline at end of file
......@@ -31,6 +31,8 @@ ENV_VARS = {
"asset_har_export_extension_crx": "CM_ASSET_HAR_EXPORT_EXTENSION_CRX",
"job_queue_delay": "CM_JOB_QUEUE_DELAY",
"fixture_location": "CM_FIXTURE_LOCATION",
"dashboard_location": "CM_DASHBOARD_LOCATION",
"dashboard_www_location": "CM_DASHBOARD_WWW_LOCATION",
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment