Merge pull request #250 from juga0/ticket27512_warn_disk_followup

import os

import logging

from sbws.util.timestamp import now_fname

from ..util.fs import is_low_space

log = logging.getLogger(__name__)

def main(args, conf):

    os.makedirs(conf.getpath('paths', 'v3bw_dname'), exist_ok=True)

    if is_low_space(conf):

        exit(1)

    datadir = conf.getpath('paths', 'datadir')

    if not os.path.isdir(datadir):

import requests

import random

from ..util.fs import is_low_space

rng = random.SystemRandom()

end_event = Event()

def main(args, conf):

    if is_low_space(conf):

        exit(1)

    if conf.getint('scanner', 'measurement_threads') < 1:

        fail_hard('Number of measurement threads must be larger than 1')

import platform

import logging

from sbws.util.fs import sbws_required_disk_space

log = logging.getLogger(__name__)

            log.critical(e)

        exit(1)

    configure_logging(args, conf)

    parser.description = sbws_required_disk_space(conf)

    def_args = [args, conf]

    def_kwargs = {}

    known_commands = {

log = logging.getLogger(__name__)

DISK_SPACE_TEXT = """

Disk space requirements

-----------------------

v3bw files: the maximum space required is ~{mb_bw} MB, after {d_bw} days.

result files: the maximum space required is ~{mb_results} MB, after {d_r} days.

tor directory: the space required is ~{mb_tor} MB.

code and depenencies: the space required is ~{mb_code} MB

Total disk space required is: ~{mb_total} MB

"""

def sbws_required_disk_space(conf):

    """Disk space required by sbws files.

    Rough calculations.

    :param ConfigParser conf: sbws configuration

    :returns: int, size in MiB

    :returns: int, size in MB

    """

    text_dict = {}

    # Number of relays per line average size in Bytes

    size_v3bw_file = 7500 * 220

    # the minimum number of required v3bw files

    num_v3bw_files = 2

    # default crontab configuration will run genenerate every hour

    num_v3bw_files_day = 24

    # ~1000 is the length of a line when the result is successfull

    # ~4550 is the number of lines of the biggest result file

    size_result_file = 4550 * 1000

    num_result_files = conf.getint('general', 'data_period')

    # not counting compressed files

    space_v3bw_files = size_v3bw_file * num_v3bw_files

    space_result_files = size_result_file * num_result_files

    num_result_files_day = 1

    space_v3bw_files_day = size_v3bw_file * num_v3bw_files_day

    space_result_files_day = size_result_file * num_result_files_day

    size_compressed_files = 600 * 1024

    # default crontab configuration will run cleanup once a day

    # default cleanup configuration will compress v3bw files after 1 day

    # and delete them after 7 days

    v3bw_compress_after_days = conf.getint('cleanup',

                                           'v3bw_files_compress_after_days')

    v3bw_delete_after_days = conf.getint('cleanup',

                                         'v3bw_files_delete_after_days')

    v3bw_max_space_after_delete = \

        (space_v3bw_files_day * v3bw_compress_after_days) + \

        (size_compressed_files * num_v3bw_files_day * v3bw_delete_after_days)

    text_dict['mb_bw'] = round(v3bw_max_space_after_delete / 1000 ** 2)

    text_dict['d_bw'] = v3bw_delete_after_days

    # default crontab configuration will run cleanup once a day

    # default cleanup configuration will compress v3bw files after 1 day

    # and delete them after 7 days

    results_compress_after_days = conf.getint('cleanup',

                                              'data_files_compress_after_days')

    results_delete_after_days = conf.getint('cleanup',

                                            'data_files_delete_after_days')

    results_max_space_after_delete = \

        (space_result_files_day * results_compress_after_days) + \

        (size_compressed_files * num_v3bw_files_day *

         results_delete_after_days)

    text_dict['mb_results'] = round(results_max_space_after_delete / 1000 ** 2)

    text_dict['d_r'] = results_delete_after_days

    # not counted rotated files and assuming that when it is not rotated the

    # size will be aproximately 10MiB

    size_log_file = conf.getint('logging', 'to_file_max_bytes') or 10485760 \

        if conf.getboolean('logging', 'to_file') else 0

    space_log_files = 0

    if conf.getboolean('logging', 'to_file'):

        size_log_file = conf.getint('logging', 'to_file_max_bytes')

        num_log_files = conf.geting('logging', 'to_file_num_backups')

        space_log_files = size_log_file * num_log_files

    text_dict['mb_log'] = space_log_files

    # roughly, size of a current tor dir

    space_tor_dir = 19828000

    # duplicate everything to warn early

    size_total = (space_v3bw_files + size_log_file + space_result_files +

                  space_tor_dir) * 2

    # convert to MiB

    size_total_mb = round(size_total / (1024 ** 2))

    return size_total_mb

    size_tor_dir = 19828000

    text_dict['mb_tor'] = round(size_tor_dir / 1000 ** 2)

    # roughly, the size of this code and dependencies

    size_code_deps = 2097152

    text_dict['mb_code'] = round(size_code_deps / 1000 ** 2)

    # Multiply per 2, just in case

    size_total = (results_max_space_after_delete +

                  v3bw_max_space_after_delete + space_log_files +

                  size_tor_dir + size_code_deps) * 2

    text_dict['mb_total'] = round(size_total / 1000 ** 2)

    space_text = DISK_SPACE_TEXT.format(**text_dict)

    return space_text

def df(path):

    # Not being used, since it makes a disk space system call and some

    # systems might not allow it

    """Return space left on device where path is in MiB."""

    return round(shutil.disk_usage(path).free / (1024 ** 2))

def is_low_space(conf):

    # Not being used, since it makes a disk space system call and some

    # systems might not allow it

    """Warn and return True when the space left on the device is less than

    what is needed for sbws and False otherwise needs.

    """

import sbws.core.stats

from sbws import __version__

from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter

from argparse import ArgumentParser, RawTextHelpFormatter

import os

def create_parser():

    p = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)

    p = ArgumentParser(formatter_class=RawTextHelpFormatter)

    p.add_argument(

        '--version', action='version', help='sbws version',

        version='{}'.format(__version__))