Loading reboot +101 −55 Original line number Diff line number Diff line #!/usr/bin/python3 # coding: utf-8 '''''' """""" # Copyright (C) 2016 Antoine Beaupré <anarcat@debian.org> # Loading Loading @@ -48,21 +48,40 @@ from fabric_tpa.reboot import ( # "sleep between hosts" policy. This probably requires overriding the # Executor class? See also https://github.com/fabric/fabric/issues/2069 def parse_args(args=sys.argv[1:]): parser = argparse.ArgumentParser(description=__doc__, epilog='''''') parser.add_argument('--quiet', '-q', dest='log_level', action='store_const', const='warning', default='info') parser.add_argument('--debug', '-d', dest='log_level', action='store_const', const='debug', default='info') parser.add_argument('--force', '-f', action='store_true', help='force reboot even if not needed') parser.add_argument('--skip-ganeti-checks', action='store_true', help='forcibly reboot Ganeti hosts without checking') parser.add_argument('--skip-ganeti-empty', action='store_true', help='reboot instances on the node as well, avoiding migrations') parser = argparse.ArgumentParser(description=__doc__, epilog="""""") parser.add_argument( '--ganeti-migrate-back', action='store_true', help='migrate the instances back after reboot', "--quiet", "-q", dest="log_level", action="store_const", const="warning", default="info", ) parser.add_argument( "--debug", "-d", dest="log_level", action="store_const", const="debug", default="info", ) parser.add_argument( "--force", "-f", action="store_true", help="force reboot even if not needed" ) parser.add_argument( "--skip-ganeti-checks", action="store_true", help="forcibly reboot Ganeti hosts without checking", ) parser.add_argument( "--skip-ganeti-empty", action="store_true", help="reboot instances on the node as well, avoiding migrations", ) parser.add_argument( "--ganeti-migrate-back", action="store_false", help="migrate the instances back after reboot", ) # TODO: autodetect from LDAP. current documented behavior is # according to rebootPolicy: Loading @@ -73,51 +92,77 @@ def parse_args(args=sys.argv[1:]): # (e.g. with hosts listed by hand here) # # "manual" seems to be the default policy parser.add_argument('--hosts', '-H', nargs='+', help="host(s) to reboot, can be comma-separated") parser.add_argument('--delay-down', default=DEFAULT_DELAY_DOWN, type=int, help='how long to wait for host to shutdown (default: %(default)s seconds)') # noqa: E501 parser.add_argument('--delay-up', default=DEFAULT_DELAY_UP, type=int, help='how long to wait for host to come back up (default: %(default)s seconds)') # noqa: E501 parser.add_argument('--delay-hosts', default=DEFAULT_DELAY_HOSTS, type=int, help='how long to wait between hosts (default: %(default)s seconds)') # noqa: E501 parser.add_argument('--delay-shutdown', default=DEFAULT_DELAY_SHUTDOWN, type=int, help='delay, in minutes, passed to the shutdown command (default: %(default)s minutes)') # noqa: E501 parser.add_argument('--reason', default='rebooting for security upgrades', help='reason to give users (default: %(default)s)') parser.add_argument('--kind', default=ShutdownType.reboot, type=ShutdownType.validator, help='kind of reboot to do (default: %(default)s)') parser.add_argument( "--hosts", "-H", nargs="+", help="host(s) to reboot, can be comma-separated" ) parser.add_argument( "--delay-down", default=DEFAULT_DELAY_DOWN, type=int, help="how long to wait for host to shutdown (default: %(default)s seconds)", ) # noqa: E501 parser.add_argument( "--delay-up", default=DEFAULT_DELAY_UP, type=int, help="how long to wait for host to come back up (default: %(default)s seconds)", ) # noqa: E501 parser.add_argument( "--delay-hosts", default=DEFAULT_DELAY_HOSTS, type=int, help="how long to wait between hosts (default: %(default)s seconds)", ) # noqa: E501 parser.add_argument( "--delay-shutdown", default=DEFAULT_DELAY_SHUTDOWN, type=int, help="delay, in minutes, passed to the shutdown command (default: %(default)s minutes)", ) # noqa: E501 parser.add_argument( "--reason", default="rebooting for security upgrades", help="reason to give users (default: %(default)s)", ) parser.add_argument( "--kind", default=ShutdownType.reboot, type=ShutdownType.validator, help="kind of reboot to do (default: %(default)s)", ) return parser.parse_args(args=args) def main(args): first = True # split each hostname on comma, like `fab -H` does for hostname in [x for h in args.hosts for x in h.split(',')]: for hostname in [x for h in args.hosts for x in h.split(",")]: con = host.find_context(hostname) logging.info("checking if host %s needs a reboot", hostname) if not needs_reboot(con): if args.force: logging.warning('rebooting anyways because of --force') logging.warning("rebooting anyways because of --force") else: logging.info('host %s does not need a reboot, skipping', hostname) logging.info("host %s does not need a reboot, skipping", hostname) continue if first: first = False else: logging.info('sleeping %d seconds before rebooting %s', args.delay_hosts, hostname) logging.info( "sleeping %d seconds before rebooting %s", args.delay_hosts, hostname ) now = datetime.now(timezone.utc) logging.info( 'now is %s, it is safe to interrupt this program until %s', "now is %s, it is safe to interrupt this program until %s", now, now + timedelta(seconds=args.delay_hosts), ) time.sleep(args.delay_hosts) delay_shutdown = args.delay_shutdown logging.info('rebooting host %s', hostname) if not shutdown_and_wait(con, logging.info("rebooting host %s", hostname) if not shutdown_and_wait( con, reason=args.reason, kind=args.kind, delay_down=args.delay_down, Loading @@ -127,31 +172,32 @@ def main(args): ganeti_empty=not args.skip_ganeti_empty, ganeti_migrate_back=args.ganeti_migrate_back, ): logging.error('rebooting host %s failed, aborting', hostname) logging.error("rebooting host %s failed, aborting", hostname) break logging.info('done with host %s', hostname) logging.info("done with host %s", hostname) # raise the bell so we bring attention to this window print('\a') print("\a") # TODO: rebalance ganeti cluster if nodes were migrated if __name__ == '__main__': if __name__ == "__main__": args = parse_args() logging.basicConfig(format='%(message)s', level=args.log_level.upper()) logging.basicConfig(format="%(message)s", level=args.log_level.upper()) # override default logging policies in submodules # # without this, we get debugging info from paramiko with --verbose for mod in 'fabric', 'paramiko', 'invoke': logging.getLogger(mod).setLevel('WARNING') for mod in "fabric", "paramiko", "invoke": logging.getLogger(mod).setLevel("WARNING") try: main(args) except Exception as e: logging.error('unexpected exception during reboot: [%r] %s', e, e) if args.log_level.upper() == 'DEBUG': logging.error("unexpected exception during reboot: [%r] %s", e, e) if args.log_level.upper() == "DEBUG": import traceback import pdb import sys traceback.print_exc() pdb.post_mortem() sys.exit(1) Loading
reboot +101 −55 Original line number Diff line number Diff line #!/usr/bin/python3 # coding: utf-8 '''''' """""" # Copyright (C) 2016 Antoine Beaupré <anarcat@debian.org> # Loading Loading @@ -48,21 +48,40 @@ from fabric_tpa.reboot import ( # "sleep between hosts" policy. This probably requires overriding the # Executor class? See also https://github.com/fabric/fabric/issues/2069 def parse_args(args=sys.argv[1:]): parser = argparse.ArgumentParser(description=__doc__, epilog='''''') parser.add_argument('--quiet', '-q', dest='log_level', action='store_const', const='warning', default='info') parser.add_argument('--debug', '-d', dest='log_level', action='store_const', const='debug', default='info') parser.add_argument('--force', '-f', action='store_true', help='force reboot even if not needed') parser.add_argument('--skip-ganeti-checks', action='store_true', help='forcibly reboot Ganeti hosts without checking') parser.add_argument('--skip-ganeti-empty', action='store_true', help='reboot instances on the node as well, avoiding migrations') parser = argparse.ArgumentParser(description=__doc__, epilog="""""") parser.add_argument( '--ganeti-migrate-back', action='store_true', help='migrate the instances back after reboot', "--quiet", "-q", dest="log_level", action="store_const", const="warning", default="info", ) parser.add_argument( "--debug", "-d", dest="log_level", action="store_const", const="debug", default="info", ) parser.add_argument( "--force", "-f", action="store_true", help="force reboot even if not needed" ) parser.add_argument( "--skip-ganeti-checks", action="store_true", help="forcibly reboot Ganeti hosts without checking", ) parser.add_argument( "--skip-ganeti-empty", action="store_true", help="reboot instances on the node as well, avoiding migrations", ) parser.add_argument( "--ganeti-migrate-back", action="store_false", help="migrate the instances back after reboot", ) # TODO: autodetect from LDAP. current documented behavior is # according to rebootPolicy: Loading @@ -73,51 +92,77 @@ def parse_args(args=sys.argv[1:]): # (e.g. with hosts listed by hand here) # # "manual" seems to be the default policy parser.add_argument('--hosts', '-H', nargs='+', help="host(s) to reboot, can be comma-separated") parser.add_argument('--delay-down', default=DEFAULT_DELAY_DOWN, type=int, help='how long to wait for host to shutdown (default: %(default)s seconds)') # noqa: E501 parser.add_argument('--delay-up', default=DEFAULT_DELAY_UP, type=int, help='how long to wait for host to come back up (default: %(default)s seconds)') # noqa: E501 parser.add_argument('--delay-hosts', default=DEFAULT_DELAY_HOSTS, type=int, help='how long to wait between hosts (default: %(default)s seconds)') # noqa: E501 parser.add_argument('--delay-shutdown', default=DEFAULT_DELAY_SHUTDOWN, type=int, help='delay, in minutes, passed to the shutdown command (default: %(default)s minutes)') # noqa: E501 parser.add_argument('--reason', default='rebooting for security upgrades', help='reason to give users (default: %(default)s)') parser.add_argument('--kind', default=ShutdownType.reboot, type=ShutdownType.validator, help='kind of reboot to do (default: %(default)s)') parser.add_argument( "--hosts", "-H", nargs="+", help="host(s) to reboot, can be comma-separated" ) parser.add_argument( "--delay-down", default=DEFAULT_DELAY_DOWN, type=int, help="how long to wait for host to shutdown (default: %(default)s seconds)", ) # noqa: E501 parser.add_argument( "--delay-up", default=DEFAULT_DELAY_UP, type=int, help="how long to wait for host to come back up (default: %(default)s seconds)", ) # noqa: E501 parser.add_argument( "--delay-hosts", default=DEFAULT_DELAY_HOSTS, type=int, help="how long to wait between hosts (default: %(default)s seconds)", ) # noqa: E501 parser.add_argument( "--delay-shutdown", default=DEFAULT_DELAY_SHUTDOWN, type=int, help="delay, in minutes, passed to the shutdown command (default: %(default)s minutes)", ) # noqa: E501 parser.add_argument( "--reason", default="rebooting for security upgrades", help="reason to give users (default: %(default)s)", ) parser.add_argument( "--kind", default=ShutdownType.reboot, type=ShutdownType.validator, help="kind of reboot to do (default: %(default)s)", ) return parser.parse_args(args=args) def main(args): first = True # split each hostname on comma, like `fab -H` does for hostname in [x for h in args.hosts for x in h.split(',')]: for hostname in [x for h in args.hosts for x in h.split(",")]: con = host.find_context(hostname) logging.info("checking if host %s needs a reboot", hostname) if not needs_reboot(con): if args.force: logging.warning('rebooting anyways because of --force') logging.warning("rebooting anyways because of --force") else: logging.info('host %s does not need a reboot, skipping', hostname) logging.info("host %s does not need a reboot, skipping", hostname) continue if first: first = False else: logging.info('sleeping %d seconds before rebooting %s', args.delay_hosts, hostname) logging.info( "sleeping %d seconds before rebooting %s", args.delay_hosts, hostname ) now = datetime.now(timezone.utc) logging.info( 'now is %s, it is safe to interrupt this program until %s', "now is %s, it is safe to interrupt this program until %s", now, now + timedelta(seconds=args.delay_hosts), ) time.sleep(args.delay_hosts) delay_shutdown = args.delay_shutdown logging.info('rebooting host %s', hostname) if not shutdown_and_wait(con, logging.info("rebooting host %s", hostname) if not shutdown_and_wait( con, reason=args.reason, kind=args.kind, delay_down=args.delay_down, Loading @@ -127,31 +172,32 @@ def main(args): ganeti_empty=not args.skip_ganeti_empty, ganeti_migrate_back=args.ganeti_migrate_back, ): logging.error('rebooting host %s failed, aborting', hostname) logging.error("rebooting host %s failed, aborting", hostname) break logging.info('done with host %s', hostname) logging.info("done with host %s", hostname) # raise the bell so we bring attention to this window print('\a') print("\a") # TODO: rebalance ganeti cluster if nodes were migrated if __name__ == '__main__': if __name__ == "__main__": args = parse_args() logging.basicConfig(format='%(message)s', level=args.log_level.upper()) logging.basicConfig(format="%(message)s", level=args.log_level.upper()) # override default logging policies in submodules # # without this, we get debugging info from paramiko with --verbose for mod in 'fabric', 'paramiko', 'invoke': logging.getLogger(mod).setLevel('WARNING') for mod in "fabric", "paramiko", "invoke": logging.getLogger(mod).setLevel("WARNING") try: main(args) except Exception as e: logging.error('unexpected exception during reboot: [%r] %s', e, e) if args.log_level.upper() == 'DEBUG': logging.error("unexpected exception during reboot: [%r] %s", e, e) if args.log_level.upper() == "DEBUG": import traceback import pdb import sys traceback.print_exc() pdb.post_mortem() sys.exit(1)
