Commit 7db1887f authored by Matt Traudt's avatar Matt Traudt Committed by juga
Browse files

Logging config overhaul ...

User should configure logging via the [logging] section
in their config.ini.

User can change the log level using --log-level, or in the config
with level, to_file_level, and to_stdout_level.

User can change the log format in the config with format,
to_file_format, and to_stdout_format

An example debugging format is included as a comment in config.default.ini

config.log.default.ini is majorly reorganized. For the most part, if
something doesn't need to be specified in this file, it has been removed.
For example, we don't have to specify any args to handler_to_file. The
beginnings of syslog configuration stuff remain as comments.

As part of configure_logging, force logging to stdout if we don't think
we can log to file.

Actually set some parameters to RotatingFileHandler so it rotates when
files reach 10 MiB and keeps 100 backups.

Each sbws command (like scanner and generate) gets its own log file. I
did this because I didn't want 'sbws scanner' to be running 24/7 and an
'sbws generate' call to log to the same file. That would get confusing.
parent 9fea5c1c
......@@ -84,11 +84,17 @@ fraction_relays = 0.05
min_relays = 50
# Level to log at. Debug, info, warning, error.
level = debug
# Whether or not to log to a rotating file the directory paths.log_dname
to_file = yes
# Whether or not to log to stdout
to_stdout = no
# Level to log at. Debug, info, warning, error, critical.
level = info
to_file_level = ${level}
to_stdout_level = ${level}
# Format string to use when logging
format = [%(asctime)s] [%(name)s] [%(levelname)s] %(message)s
to_file_format = ${format}
to_stdout_format = ${format}
# verbose formatter useful for debugging
#format = %(asctime)s %(levelname)s %(threadName)s %(filename)s:%(lineno)s - %(funcName)s - %(message)s
......@@ -2,62 +2,49 @@
keys = root,sbws
keys = sbwsdefault,sbwsdebug,sbwsfile
keys = to_file,to_stdout
keys = sbwsdefault,sbwsdebug,sbwssys
keys = to_file,to_stdout
level = WARNING
handlers = sbwsdefault
handlers = to_file
propagate = 1
level = INFO
# add here sbwssys for also logging to system log
handlers = sbwsdefault,sbwsfile
propagate = 0
class = StreamHandler
formatter = sbwsdefault
formatter = to_stdout
args = (sys.stdout,)
format = [%(asctime)s] [%(name)s] [%(levelname)s] %(message)s
# this is not needed, but here to do not modify sbwsdefault
class = StreamHandler
formatter = sbwsdebug
args = (sys.stdout,)
level = DEBUG
# verbose formatter useful for debugging
format = %(asctime)s %(levelname)s %(threadName)s %(filename)s:%(lineno)s - %(funcName)s - %(message)s
datefmt = %Y-%m-%d %H:%M:%S
# for logging to system log
level = INFO
# syslog-like formater
# hostname should be added here with a context filter, dunno if there is a way
# to add it here
format = %(asctime)s %(module)s[%(process)s]: <%(levelname)s> %(message)s
datefmt = %b %d %H:%M:%S
class = handlers.RotatingFileHandler
formatter = sbwssys
formatter = to_file
# There doesn't seem to be a way to put ${paths:log_filepath} here since the
# logging library eval()s this, which doesn't give the ini parsing library a
# chance to replace the text. So we do it in configure_logging() in
# sbws/util/ instead.
args = ('/nonexist',)
args =
## for logging to system log
#level = INFO
#args = ()
## syslog-like formater
## hostname should be added here with a context filter, dunno if there is a way
## to add it here
#format = %(asctime)s %(module)s[%(process)s]: <%(levelname)s> %(message)s
#datefmt = %b %d %H:%M:%S
......@@ -28,7 +28,7 @@ def main():
for e in conf_errors:
# configure_logging(conf)
configure_logging(args, conf)
def_args = [args, conf]
def_kwargs = {}
known_commands = {
from configparser import (ConfigParser, ExtendedInterpolation)
from configparser import InterpolationMissingOptionError
import os
import logging
import logging.config
......@@ -15,6 +16,8 @@ _SYMBOLS_NO_QUOTES = '!@#$%^&*()-_=+\\|[]{}:;/?.,<>'
_HEX = '0123456789ABCDEF'
_LOG_LEVELS = ['debug', 'info', 'warning', 'error', 'critical']
log = logging.getLogger(__name__)
......@@ -72,12 +75,63 @@ def get_user_example_config():
return conf
def configure_logging(conf):
def _can_log_to_file(conf):
Checks all the known reasons for why we might not be able to log to a file,
and returns whether or not we think we will be able to do so. This is
useful because if we can't log to a file, we might want to force logging to
If we can't log to file, return False and the reason. Otherwise return True
and an empty string.
# We won't be able to get paths.log_dname from the config when we are first
# initializing sbws because it depends on paths.sbws_home (by default).
# If there is an issue getting this option, tell the caller that we can't
# log to file.
except InterpolationMissingOptionError as e:
return False, e
return True, ''
def configure_logging(args, conf):
assert isinstance(conf, ConfigParser)
# log_filepath is not a variable in config.log.default.ini,
# so adding it here. Maybe there is a better way to do this
conf['handler_sbwsfile']['args'] = \
logger = 'logger_sbws'
if args.log_level:
conf[logger]['level'] = args.log_level.upper()
# Set the correct handler(s) based on [logging] options
handlers = set()
can_log_to_file, reason = _can_log_to_file(conf)
if not can_log_to_file or conf.getboolean('logging', 'to_stdout'):
# always add to_stdout if we cannot log to file
if can_log_to_file and conf.getboolean('logging', 'to_file'):
# Collect the handlers in the appropriate config option
conf[logger]['handlers'] = ','.join(handlers)
if 'to_file' in handlers:
# Because of the way python's standard logging library works, we can't
# tell this handler that the file it should log to is
# ${paths:log_dname}/foo.log. It evals() the string stored in the args,
# therefore not allowing the config parser to change that to
# ~/.sbws/log/foo.log. So we set it here.
# Also we set files to rotate at 10 MiB in size and to keep 100 backups
dname = conf['paths']['log_dname']
os.makedirs(dname, exist_ok=True)
fname = os.path.join(dname, '{}.log'.format(args.command or 'sbws'))
conf['handler_to_file']['args'] = \
"('{}', 'a', 10*1024*1024, 100)".format(fname)
# Set some stuff that needs config parser's interpolation
conf['formatter_to_file']['format'] = conf['logging']['to_file_format']
conf['formatter_to_stdout']['format'] = conf['logging']['to_stdout_format']
conf[logger]['level'] = conf['logging']['level'].upper()
conf['handler_to_file']['level'] = conf['logging']['to_file_level'].upper()
conf['handler_to_stdout']['level'] = \
# Now we configure the standard python logging system
with NamedTemporaryFile('w+t') as fd:
conf.write(fd), 0)
......@@ -220,13 +274,15 @@ def _validate_logging(conf):
sec = 'logging'
err_tmpl = Template('$sec/$key ($val): $e')
enums = {
'level': {'choices': ['debug', 'info', 'warning', 'error']},
'level': {'choices': _LOG_LEVELS},
'to_file_level': {'choices': _LOG_LEVELS},
'to_stdout_level': {'choices': _LOG_LEVELS},
bools = {
'to_file': {},
'to_stdout': {},
unvalidated = ['format']
unvalidated = ['format', 'to_file_format', 'to_stdout_format']
all_valid_keys = list(bools.keys()) + list(enums.keys()) + unvalidated
errors.extend(_validate_section_keys(conf, sec, all_valid_keys, err_tmpl))
errors.extend(_validate_section_bools(conf, sec, bools, err_tmpl))
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment