Unverified Commit aea806bf authored by juga's avatar juga Committed by GitHub
Browse files

Merge pull request #250 from juga0/ticket27512_warn_disk_followup

Ticket27512 warn disk followup
parents 75be1aa2 1ba79e55
......@@ -5,7 +5,6 @@ from argparse import ArgumentDefaultsHelpFormatter
import os
import logging
from sbws.util.timestamp import now_fname
from ..util.fs import is_low_space
log = logging.getLogger(__name__)
......@@ -41,8 +40,6 @@ def gen_parser(sub):
def main(args, conf):
os.makedirs(conf.getpath('paths', 'v3bw_dname'), exist_ok=True)
if is_low_space(conf):
exit(1)
datadir = conf.getpath('paths', 'datadir')
if not os.path.isdir(datadir):
......
......@@ -21,8 +21,6 @@ import logging
import requests
import random
from ..util.fs import is_low_space
rng = random.SystemRandom()
end_event = Event()
......@@ -371,9 +369,6 @@ def gen_parser(sub):
def main(args, conf):
if is_low_space(conf):
exit(1)
if conf.getint('scanner', 'measurement_threads') < 1:
fail_hard('Number of measurement threads must be larger than 1')
......
......@@ -14,6 +14,8 @@ from requests.__version__ import __version__ as requests_version
import platform
import logging
from sbws.util.fs import sbws_required_disk_space
log = logging.getLogger(__name__)
......@@ -51,6 +53,7 @@ def main():
log.critical(e)
exit(1)
configure_logging(args, conf)
parser.description = sbws_required_disk_space(conf)
def_args = [args, conf]
def_kwargs = {}
known_commands = {
......
......@@ -5,45 +5,94 @@ import shutil
log = logging.getLogger(__name__)
DISK_SPACE_TEXT = """
Disk space requirements
-----------------------
v3bw files: the maximum space required is ~{mb_bw} MB, after {d_bw} days.
result files: the maximum space required is ~{mb_results} MB, after {d_r} days.
tor directory: the space required is ~{mb_tor} MB.
code and depenencies: the space required is ~{mb_code} MB
Total disk space required is: ~{mb_total} MB
"""
def sbws_required_disk_space(conf):
"""Disk space required by sbws files.
Rough calculations.
:param ConfigParser conf: sbws configuration
:returns: int, size in MiB
:returns: int, size in MB
"""
text_dict = {}
# Number of relays per line average size in Bytes
size_v3bw_file = 7500 * 220
# the minimum number of required v3bw files
num_v3bw_files = 2
# default crontab configuration will run genenerate every hour
num_v3bw_files_day = 24
# ~1000 is the length of a line when the result is successfull
# ~4550 is the number of lines of the biggest result file
size_result_file = 4550 * 1000
num_result_files = conf.getint('general', 'data_period')
# not counting compressed files
space_v3bw_files = size_v3bw_file * num_v3bw_files
space_result_files = size_result_file * num_result_files
num_result_files_day = 1
space_v3bw_files_day = size_v3bw_file * num_v3bw_files_day
space_result_files_day = size_result_file * num_result_files_day
size_compressed_files = 600 * 1024
# default crontab configuration will run cleanup once a day
# default cleanup configuration will compress v3bw files after 1 day
# and delete them after 7 days
v3bw_compress_after_days = conf.getint('cleanup',
'v3bw_files_compress_after_days')
v3bw_delete_after_days = conf.getint('cleanup',
'v3bw_files_delete_after_days')
v3bw_max_space_after_delete = \
(space_v3bw_files_day * v3bw_compress_after_days) + \
(size_compressed_files * num_v3bw_files_day * v3bw_delete_after_days)
text_dict['mb_bw'] = round(v3bw_max_space_after_delete / 1000 ** 2)
text_dict['d_bw'] = v3bw_delete_after_days
# default crontab configuration will run cleanup once a day
# default cleanup configuration will compress v3bw files after 1 day
# and delete them after 7 days
results_compress_after_days = conf.getint('cleanup',
'data_files_compress_after_days')
results_delete_after_days = conf.getint('cleanup',
'data_files_delete_after_days')
results_max_space_after_delete = \
(space_result_files_day * results_compress_after_days) + \
(size_compressed_files * num_v3bw_files_day *
results_delete_after_days)
text_dict['mb_results'] = round(results_max_space_after_delete / 1000 ** 2)
text_dict['d_r'] = results_delete_after_days
# not counted rotated files and assuming that when it is not rotated the
# size will be aproximately 10MiB
size_log_file = conf.getint('logging', 'to_file_max_bytes') or 10485760 \
if conf.getboolean('logging', 'to_file') else 0
space_log_files = 0
if conf.getboolean('logging', 'to_file'):
size_log_file = conf.getint('logging', 'to_file_max_bytes')
num_log_files = conf.geting('logging', 'to_file_num_backups')
space_log_files = size_log_file * num_log_files
text_dict['mb_log'] = space_log_files
# roughly, size of a current tor dir
space_tor_dir = 19828000
# duplicate everything to warn early
size_total = (space_v3bw_files + size_log_file + space_result_files +
space_tor_dir) * 2
# convert to MiB
size_total_mb = round(size_total / (1024 ** 2))
return size_total_mb
size_tor_dir = 19828000
text_dict['mb_tor'] = round(size_tor_dir / 1000 ** 2)
# roughly, the size of this code and dependencies
size_code_deps = 2097152
text_dict['mb_code'] = round(size_code_deps / 1000 ** 2)
# Multiply per 2, just in case
size_total = (results_max_space_after_delete +
v3bw_max_space_after_delete + space_log_files +
size_tor_dir + size_code_deps) * 2
text_dict['mb_total'] = round(size_total / 1000 ** 2)
space_text = DISK_SPACE_TEXT.format(**text_dict)
return space_text
def df(path):
# Not being used, since it makes a disk space system call and some
# systems might not allow it
"""Return space left on device where path is in MiB."""
return round(shutil.disk_usage(path).free / (1024 ** 2))
def is_low_space(conf):
# Not being used, since it makes a disk space system call and some
# systems might not allow it
"""Warn and return True when the space left on the device is less than
what is needed for sbws and False otherwise needs.
"""
......
......@@ -4,7 +4,7 @@ import sbws.core.generate
import sbws.core.stats
from sbws import __version__
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from argparse import ArgumentParser, RawTextHelpFormatter
import os
......@@ -14,7 +14,7 @@ def _default_dot_sbws_dname():
def create_parser():
p = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
p = ArgumentParser(formatter_class=RawTextHelpFormatter)
p.add_argument(
'--version', action='version', help='sbws version',
version='{}'.format(__version__))
......
"""Unit tests for fs module"""
from unittest.mock import patch
from sbws.util import fs
def mock_df_zero(path):
return 0
def mock_df_enough(path):
return 89
@patch('sbws.util.fs.df', mock_df_zero)
def test_is_low_space_true(caplog, conf):
assert fs.is_low_space(conf) is True
assert ' is less than ' in caplog.records[-1].getMessage()
@patch('sbws.util.fs.df', mock_df_enough)
def test_is_low_space_false(conf):
assert fs.is_low_space(conf) is False
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment