#!/usr/bin/env python
# -*- mode: python -*-
# Generates passwd, shadow and group files from the ldap directory.

#   Copyright (c) 2000-2001  Jason Gunthorpe <jgg@debian.org>
#   Copyright (c) 2003-2004  James Troup <troup@debian.org>
#   Copyright (c) 2004-2005,7  Joey Schulze <joey@infodrom.org>
#   Copyright (c) 2001-2007  Ryan Murray <rmurray@debian.org>
#   Copyright (c) 2008,2009,2010,2011 Peter Palfrader <peter@palfrader.org>
#   Copyright (c) 2008 Andreas Barth <aba@not.so.argh.org>
#   Copyright (c) 2008 Mark Hymers <mhy@debian.org>
#   Copyright (c) 2008 Luk Claes <luk@debian.org>
#   Copyright (c) 2008 Thomas Viehmann <tv@beamnet.de>
#   Copyright (c) 2009 Stephen Gran <steve@lobefin.net>
#   Copyright (c) 2010 Helmut Grohne <helmut@subdivi.de>
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

from __future__ import print_function

import string
import re
import time
import optparse
import sys
import os
import pwd
import posix
import socket
import base64
import hashlib
import shutil
import errno
import tarfile
import grp
import fcntl
import dbm
import subprocess

from xml.etree import ElementTree
from xml.dom import minidom
try:
   from cStringIO import StringIO
except ImportError:
   from StringIO import StringIO
import json

import ldap

from dsa_mq.connection import Connection
from dsa_mq.config import Config

from userdir_ldap import (
    GetAttr, AllowedGroupsPreload, HomePrefix, connectLDAP, HostBaseDn, BaseDn,
    HostDomain, GenerateDir, ConfModule, UserDNSDomain, make_passwd_hmac,
    FormatPGPKey, PassDir)
from userdir_exceptions import UDEmptyList
import UDLdap

if os.getuid() == 0:
   sys.stderr.write("You should probably not run ud-generate as root.\n")
   sys.exit(1)


#
# GLOBAL STATE
#
GroupIDMap = None
SubGroupMap = None


UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
MAX_UD_AGE = 3600 * 24

EmailCheck = re.compile(r"^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$")
BSMTPCheck = re.compile(r".*mx 0 (master)\.debian\.org\..*", re.DOTALL)
PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
IsDebianHost = re.compile(ConfModule.dns_hostmatch)
isSSHFP = re.compile(r"^\s*IN\s+SSHFP")
if UserDNSDomain:
   DNSZone = "." + UserDNSDomain
else:
   DNSZone = ".debian.net"
DKIMSuffix = "_domainkey.debian.org"
Keyrings = ConfModule.sync_keyrings.split(":")
GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)
GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
MX_remap = json.loads(ConfModule.MX_remap)
use_mq = getattr(ConfModule, "use_mq", True)

rtc_realm = getattr(ConfModule, "rtc_realm", None)
rtc_append = getattr(ConfModule, "rtc_append", None)


def prettify(elem):
   """Return a pretty-printed XML string for the Element.
   """
   rough_string = ElementTree.tostring(elem, 'utf-8')
   reparsed = minidom.parseString(rough_string)
   return reparsed.toprettyxml(indent="  ")


def safe_makedirs(dir):
   try:
      os.makedirs(dir)
   except OSError as e:
      if e.errno == errno.EEXIST:
         pass
      else:
         raise e


def safe_rmtree(dir):
   try:
      shutil.rmtree(dir)
   except OSError as e:
      if e.errno == errno.ENOENT:
         pass
      else:
         raise e


def get_lock(fn, wait=5 * 60):
   f = open(fn, "w")
   sl = 0.1
   ends = time.time() + wait

   while True:
      try:
         fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
         return f
      except IOError:
         pass
      if time.time() >= ends:
         return None
      sl = min(sl * 2, 10, ends - time.time())
      time.sleep(sl)
   return None


def Sanitize(Str):
   return Str.translate(string.maketrans("\n\r\t", "$$$"))


def DoLink(From, To, File):
   try:
      posix.remove(To + File)
   except Exception:
      pass
   posix.link(From + File, To + File)


def IsRetired(account):
   """
   Looks for accountStatus in the LDAP record and tries to
   match it against one of the known retired statuses
   """

   status = account['accountStatus']

   line = status.split()
   status = line[0]

   if status == "inactive":
      return True

   elif status == "memorial":
      return True

   elif status == "retiring":
      # We'll give them a few extra days over what we said
      age = 6 * 31 * 24 * 60 * 60
      try:
         return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age
      except IndexError:
         return False
      except ValueError:
         return False

   return False


# See if this user is in the group list
def IsInGroup(account, allowed, current_host):
  # See if the primary group is in the list
  if str(account['gidNumber']) in allowed:
      return True

  # Check the host based ACL
  if account.is_allowed_by_hostacl(current_host):
      return True

  # See if there are supplementary groups
  if 'supplementaryGid' not in account:
      return False

  supgroups = []
  addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)
  for g in supgroups:
     if g in allowed:
        return True
  return False


def Die(File, F, Fdb):
   if F is not None:
      F.close()
   if Fdb is not None:
      Fdb.close()
   try:
      os.remove(File + ".tmp")
   except Exception:
      pass
   try:
      os.remove(File + ".tdb.tmp")
   except Exception:
      pass


def Done(File, F, Fdb):
   if F is not None:
      F.close()
      os.rename(File + ".tmp", File)
   if Fdb is not None:
      Fdb.close()
      os.rename(File + ".tdb.tmp", File + ".tdb")


# Generate the password list
def GenPasswd(accounts, File, HomePrefix, PwdMarker):
   F = None
   try:
      F = open(File + ".tdb.tmp", "w")

      userlist = {}
      i = 0
      for a in accounts:
         if 'loginShell' not in a:
             continue
         # Do not let people try to buffer overflow some busted passwd parser.
         if len(a['gecos']) > 100 or len(a['loginShell']) > 50:
            continue

         userlist[a['uid']] = a['gidNumber']
         line = "%s:%s:%d:%d:%s:%s%s:%s" % (
             a['uid'],
             PwdMarker,
             a['uidNumber'],
             a['gidNumber'],
             a['gecos'],
             HomePrefix, a['uid'],
             a['loginShell'])
         line = Sanitize(line) + "\n"
         F.write("0%u %s" % (i, line))
         F.write(".%s %s" % (a['uid'], line))
         F.write("=%d %s" % (a['uidNumber'], line))
         i = i + 1

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, None, F)
      raise
   Done(File, None, F)

   # Return the list of users so we know which keys to export
   return userlist


def GenAllUsers(accounts, file):
   f = None
   try:
      OldMask = os.umask(0o022)
      f = open(file + ".tmp", "w")
      os.umask(OldMask)

      all = []
      for a in accounts:
         all.append({'uid': a['uid'],
                     'uidNumber': a['uidNumber'],
                     'active': a.pw_active() and a.shadow_active()})
      json.dump(all, f)

   # Oops, something unspeakable happened.
   except Exception:
      Die(file, f, None)
      raise
   Done(file, f, None)


# Generate the shadow list
def GenShadow(accounts, File):
   F = None
   try:
      OldMask = os.umask(0o077)
      F = open(File + ".tdb.tmp", "w")
      os.umask(OldMask)

      i = 0
      for a in accounts:
         # If the account is locked, mark it as such in shadow
         # See Debian Bug #308229 for why we set it to 1 instead of 0
         if not a.pw_active():
            ShadowExpire = '1'
         elif 'shadowExpire' in a:
            ShadowExpire = str(a['shadowExpire'])
         else:
            ShadowExpire = ''

         values = []
         values.append(a['uid'])
         values.append(a.get_password())
         for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive':
            if key in a:
               values.append(a[key])
            else:
               values.append('')
         values.append(ShadowExpire)
         line = ':'.join(values) + ':'
         line = Sanitize(line) + "\n"
         F.write("0%u %s" % (i, line))
         F.write(".%s %s" % (a['uid'], line))
         i = i + 1

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, None, F)
      raise
   Done(File, None, F)


# Generate the sudo passwd file
def GenShadowSudo(accounts, File, untrusted, current_host):
   F = None
   try:
      OldMask = os.umask(0o077)
      F = open(File + ".tmp", "w")
      os.umask(OldMask)

      for a in accounts:
         Pass = '*'
         if 'sudoPassword' in a:
            for entry in a['sudoPassword']:
               Match = re.compile('^(' + UUID_FORMAT + ') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
               if Match is None:
                  continue
               uuid = Match.group(1)
               status = Match.group(2)
               hosts = Match.group(3)
               cryptedpass = Match.group(4)

               if status != 'confirmed:' + make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass):
                  continue
               for_all = hosts == "*"
               for_this_host = current_host in hosts.split(',')
               if not (for_all or for_this_host):
                  continue
               # ignore * passwords for untrusted hosts, but copy host specific passwords
               if for_all and untrusted:
                  continue
               Pass = cryptedpass
               if for_this_host:  # this makes sure we take a per-host entry over the for-all entry
                  break
            if len(Pass) > 50:
               Pass = '*'

         Line = "%s:%s" % (a['uid'], Pass)
         Line = Sanitize(Line) + "\n"
         F.write("%s" % (Line))

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate the gitolite SSH authorized keys file
def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):
   F = None
   if sshcommand is None:
      sshcommand = GitoliteSSHCommand
   try:
      OldMask = os.umask(0o022)
      F = open(File + ".tmp", "w")
      os.umask(OldMask)

      if GitoliteSSHRestrictions is not None and GitoliteSSHRestrictions != "":
         for a in accounts:
            if 'sshRSAAuthKey' not in a:
               continue

            User = a['uid']
            prefix = GitoliteSSHRestrictions
            prefix = prefix.replace('@@COMMAND@@', sshcommand)
            prefix = prefix.replace('@@USER@@', User)
            for key in a["sshRSAAuthKey"]:
               if key.startswith("allowed_hosts=") and ' ' in key:
                  if current_host is None:
                     continue
                  machines, key = key.split('=', 1)[1].split(' ', 1)
                  if current_host not in machines.split(','):
                     continue  # skip this key

               if key.startswith('ssh-'):
                  line = "%s %s" % (prefix, key)
               else:
                  continue  # do not allow keys with other restrictions that might conflict
               line = Sanitize(line) + "\n"
               F.write(line)

         for dn, attrs in hosts:
            if 'sshRSAHostKey' not in attrs:
               continue
            hostname = "host-" + attrs['hostname'][0]
            prefix = GitoliteSSHRestrictions
            prefix = prefix.replace('@@COMMAND@@', sshcommand)
            prefix = prefix.replace('@@USER@@', hostname)
            for key in attrs["sshRSAHostKey"]:
               line = "%s %s" % (prefix, key)
               line = Sanitize(line) + "\n"
               F.write(line)

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate the shadow list
def GenSSHShadow(global_dir, accounts):
   # Fetch all the users
   userkeys = {}

   for a in accounts:
      if 'sshRSAAuthKey' not in a:
         continue

      contents = []
      for key in a['sshRSAAuthKey']:
         MultipleLine = "%s" % key
         MultipleLine = Sanitize(MultipleLine)
         contents.append(MultipleLine)
      userkeys[a['uid']] = contents
   return userkeys


# Generate the webPassword list
def GenWebPassword(accounts, File):
   F = None
   try:
      OldMask = os.umask(0o077)
      F = open(File, "w")
      os.umask(OldMask)

      for a in accounts:
         if 'webPassword' not in a:
            continue
         if not a.pw_active():
            continue

         Pass = str(a['webPassword'])
         Line = "%s:%s" % (a['uid'], Pass)
         Line = Sanitize(Line) + "\n"
         F.write("%s" % (Line))

   except Exception:
      Die(File, None, F)
      raise


# Generate the rtcPassword list
def GenRtcPassword(accounts, File):
   F = None
   try:
      OldMask = os.umask(0o077)
      F = open(File, "w")
      os.umask(OldMask)

      for a in accounts:
         if a.is_guest_account():
            continue
         if 'rtcPassword' not in a:
            continue
         if not a.pw_active():
            continue

         Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm)
         Line = Sanitize(Line) + "\n"
         F.write("%s" % (Line))

   except Exception:
      Die(File, None, F)
      raise


# Generate the TOTP auth file
def GenTOTPSeed(accounts, File):
   F = None
   try:
      OldMask = os.umask(0o077)
      F = open(File, "w")
      os.umask(OldMask)

      F.write("# Option User Prefix Seed\n")
      for a in accounts:
         if a.is_guest_account():
            continue
         if 'totpSeed' not in a:
            continue
         if not a.pw_active():
            continue

         Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed'])
         Line = Sanitize(Line) + "\n"
         F.write("%s" % (Line))
   except Exception:
      Die(File, None, F)
      raise


def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
   OldMask = os.umask(0o077)
   tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
   os.umask(OldMask)
   for f in userlist:
      if f not in ssh_userkeys:
         continue
      # If we're not exporting their primary group, don't export
      # the key and warn
      grname = None
      if userlist[f] in grouprevmap.keys():
         grname = grouprevmap[userlist[f]]
      else:
         try:
            if int(userlist[f]) <= 100:
               # In these cases, look it up in the normal way so we
               # deal with cases where, for instance, users are in group
               # users as their primary group.
               grname = grp.getgrgid(userlist[f])[0]
         except Exception:
            pass

      if grname is None:
         print("User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f]))
         continue

      lines = []
      for line in ssh_userkeys[f]:
         if line.startswith("allowed_hosts=") and ' ' in line:
            machines, line = line.split('=', 1)[1].split(' ', 1)
            if current_host not in machines.split(','):
               continue  # skip this key
         lines.append(line)
      if not lines:
         continue  # no keys for this host
      contents = "\n".join(lines) + "\n"

      to = tarfile.TarInfo(name=f)
      # These will only be used where the username doesn't
      # exist on the target system for some reason; hence,
      # in those cases, the safest thing is for the file to
      # be owned by root but group nobody.  This deals with
      # the bloody obscure case where the group fails to exist
      # whilst the user does (in which case we want to avoid
      # ending up with a file which is owned user:root to avoid
      # a fairly obvious attack vector)
      to.uid = 0
      to.gid = 65534
      # Using the username / groupname fields avoids any need
      # to give a shit^W^W^Wcare about the UIDoffset stuff.
      to.uname = f
      to.gname = grname
      to.mode = 0o400
      to.mtime = int(time.time())
      to.size = len(contents)

      tf.addfile(to, StringIO(contents))

   tf.close()
   os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target)


# add a list of groups to existing groups,
# including all subgroups thereof, recursively.
# basically this proceduces the transitive hull of the groups in
# addgroups.
def addGroups(existingGroups, newGroups, uid, current_host):
   for group in newGroups:
      # if it's a <group>@host, split it and verify it's on the current host.
      s = group.split('@', 1)
      if len(s) == 2 and s[1] != current_host:
         continue
      group = s[0]

      # let's see if we handled this group already
      if group in existingGroups:
         continue

      if group not in GroupIDMap:
         print("Group", group, "does not exist but", uid, "is in it")
         continue

      existingGroups.append(group)

      if group in SubGroupMap:
         addGroups(existingGroups, SubGroupMap[group], uid, current_host)


# Generate the group list
def GenGroup(accounts, File, current_host):
   grouprevmap = {}
   F = None
   try:
      F = open(File + ".tdb.tmp", "w")

      # Generate the GroupMap
      GroupMap = {}
      for x in GroupIDMap:
         GroupMap[x] = []
      GroupHasPrimaryMembers = {}

      # Sort them into a list of groups having a set of users
      for a in accounts:
         GroupHasPrimaryMembers[a['gidNumber']] = True
         if 'supplementaryGid' not in a:
            continue

         supgroups = []
         addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
         for g in supgroups:
            GroupMap[g].append(a['uid'])

      # Output the group file.
      J = 0
      for x in GroupMap.keys():
         if x not in GroupIDMap:
            continue

         if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
            continue

         grouprevmap[GroupIDMap[x]] = x

         Line = "%s:x:%u:" % (x, GroupIDMap[x])
         Comma = ''
         for user in GroupMap[x]:
            Line = Line + ("%s%s" % (Comma, user))
            Comma = ','
         Line = Sanitize(Line) + "\n"
         F.write("0%u %s" % (J, Line))
         F.write(".%s %s" % (x, Line))
         F.write("=%u %s" % (GroupIDMap[x], Line))
         J = J + 1

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, None, F)
      raise
   Done(File, None, F)

   return grouprevmap


def CheckForward(accounts):
   for a in accounts:
      if 'emailForward' not in a:
         continue

      delete = False

      # Do not allow people to try to buffer overflow busted parsers
      if len(a['emailForward']) > 200:
         delete = True
      # Check the forwarding address
      elif EmailCheck.match(a['emailForward']) is None:
         delete = True

      if delete:
         a.delete_mailforward()


# Generate the email forwarding list
def GenForward(accounts, File):
   F = None
   try:
      OldMask = os.umask(0o022)
      F = open(File + ".tmp", "w")
      os.umask(OldMask)

      for a in accounts:
         if 'emailForward' not in a:
            continue
         Line = "%s: %s" % (a['uid'], a['emailForward'])
         Line = Sanitize(Line) + "\n"
         F.write(Line)

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


def GenCDB(accounts, File, key):
   prefix = ["/usr/bin/eatmydata"] if os.path.exists('/usr/bin/eatmydata') else []
   # nothing else does the fsync stuff, so why do it here?
   Fdb = subprocess.Popen(prefix + ["cdbmake", File, "%s.tmp" % File],
                          preexec_fn=lambda: os.umask(0o022),
                          stdin=subprocess.PIPE)
   try:
      # Write out the email address for each user
      for a in accounts:
         if key not in a:
            continue
         value = a[key]
         user = a['uid']
         Fdb.stdin.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))

      Fdb.stdin.write("\n")
   finally:
      Fdb.stdin.close()
      if Fdb.wait() != 0:
         raise Exception("cdbmake gave an error")


def GenDBM(accounts, File, key):
   Fdb = None
   OldMask = os.umask(0o022)
   fn = os.path.join(File).encode('ascii', 'ignore')
   try:
      posix.remove(fn)
   except Exception:
      pass

   try:
      Fdb = dbm.open(fn, "c")
      os.umask(OldMask)

      # Write out the email address for each user
      for a in accounts:
         if key not in a:
            continue
         value = a[key]
         user = a['uid']
         Fdb[user] = value

      Fdb.close()
   except Exception:
      # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
      os.remove(File + ".db")
      raise
   # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
   os.rename(File + ".db", File)


# Generate the anon XEarth marker file
def GenMarkers(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Write out the position for each user
      for a in accounts:
         if not ('latitude' in a and 'longitude' in a):
            continue
         try:
            Line = "%8s %8s \"\"" % (a.latitude_dec(True), a.longitude_dec(True))
            Line = Sanitize(Line) + "\n"
            F.write(Line)
         except Exception:
            pass

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate the debian-private subscription list
def GenPrivate(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Write out the position for each user
      for a in accounts:
         if not a.is_active_user():
            continue
         if a.is_guest_account():
            continue
         if 'privateSub' not in a:
            continue
         try:
            Line = "%s" % a['privateSub']
            Line = Sanitize(Line) + "\n"
            F.write(Line)
         except Exception:
            pass

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate a list of locked accounts
def GenDisabledAccounts(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")
      disabled_accounts = []

      # Fetch all the users
      for a in accounts:
         if a.pw_active():
            continue
         Line = "%s:%s" % (a['uid'], "Account is locked")
         disabled_accounts.append(a)
         F.write(Sanitize(Line) + "\n")

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)
   return disabled_accounts


# Generate the list of local addresses that refuse all mail
def GenMailDisable(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      for a in accounts:
         if 'mailDisableMessage' not in a:
            continue
         Line = "%s: %s" % (a['uid'], a['mailDisableMessage'])
         Line = Sanitize(Line) + "\n"
         F.write(Line)

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate a list of uids that should have boolean affects applied
def GenMailBool(accounts, File, key):
   F = None
   try:
      F = open(File + ".tmp", "w")

      for a in accounts:
         if key not in a:
            continue
         if not a[key] == 'TRUE':
            continue
         Line = "%s" % a['uid']
         Line = Sanitize(Line) + "\n"
         F.write(Line)

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate a list of hosts for RBL or whitelist purposes.
def GenMailList(accounts, File, key):
   F = None
   try:
      F = open(File + ".tmp", "w")

      if key == "mailWhitelist":
         validregex = re.compile(r'^[-\w.]+(/[\d]+)?$')
      else:
         validregex = re.compile(r'^[-\w.]+$')

      for a in accounts:
         if key not in a:
            continue

         filtered = filter(lambda z: validregex.match(z), a[key])
         if len(filtered) == 0:
            continue
         if key == "mailRHSBL":
            filtered = map(lambda z: z + "/$sender_address_domain", filtered)
         line = a['uid'] + ': ' + ' : '.join(filtered)
         line = Sanitize(line) + "\n"
         F.write(line)

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


def isRoleAccount(account):
   return 'debianRoleAccount' in account['objectClass']


# Generate the DNS Zone file
def GenDNS(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Fetch all the users
      RRs = {}

      # Write out the zone file entry for each user
      for a in accounts:
         if 'dnsZoneEntry' not in a:
            continue
         if not a.is_active_user() and not isRoleAccount(a):
            continue
         if a.is_guest_account():
            continue

         try:
            F.write("; %s\n" % a.email_address())
            for z in a["dnsZoneEntry"]:
               Split = z.lower().split()
               if Split[1].lower() == 'in':
                  if Split[2].lower() == 'txt':
                     # TXT records likely have embedded whitespace that should
                     # be retained
                     Line = "%s \"%s\"\n" % (" ".join(Split[0:3]), " ".join(Split[3:]))
                  else:
                     Line = " ".join(Split) + "\n"
                  F.write(Line)

                  Host = Split[0] + DNSZone
                  if BSMTPCheck.match(Line) is not None:
                     F.write("; Has BSMTP\n")

                  # Write some identification information
                  if Host not in RRs:
                     if Split[2].lower() in ["a", "aaaa"]:
                        Line = "%s IN TXT \"%s\"\n" % (Split[0], a.email_address())
                        for y in a["keyFingerPrint"]:
                           Line = Line + "%s IN TXT \"PGP %s\"\n" % (Split[0], FormatPGPKey(y))
                           F.write(Line)
                        RRs[Host] = 1
               else:
                  Line = "; Err %s" % str(Split)
                  F.write(Line)

            F.write("\n")
         except Exception as e:
            F.write("; Errors:\n")
            for line in str(e).split("\n"):
               F.write("; %s\n" % line)
            pass

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate the DKIM records, appending to an open file handle
def GenDKIM(accounts, F):
   F.write('\n; DKIM\n\n')

   ok_attrs = ['h', 'k']

   # Write out the zone file entry for each user
   for a in accounts:
      if 'dkimPubKey' not in a:
         continue
      if not a.is_active_user() and not isRoleAccount(a):
         continue
      if a.is_guest_account():
         continue

      try:
         for z in a["dkimPubKey"]:
             # selector, pubkey
             Split = z.split()
             keyattrs = {'h': 'sha256', 'k': 'rsa'}
             selector = Split[0]
             if len(Split) == 3 and Split[1] == 'CNAME':
                F.write("%s.%s.\tIN\tCNAME\t%s\n" % (selector, DKIMSuffix, Split[2]))
                continue
             if len(Split) == 2:
                pubkey = Split[1]
             else:
                for attr in Split[1:]:
                   k, v = attr.split('=', 1)
                   if v.endswith(';'):
                      v = v[:-1]
                   if k in ok_attrs:
                      keyattrs[k] = v
                   elif k == 'p':
                      pubkey = v
             if pubkey.startswith('p='):
                pubkey = pubkey[2:]
             Line = "%s.%s.\tIN\tTXT (\"v=DKIM1; k=%s; s=email; h=%s; p=\"\n" % (selector, DKIMSuffix, keyattrs['k'], keyattrs['h'])
             F.write(Line)
             # A single string in a TXT RR cannot be longer than 255 characters.
             # DKIM keys will routinely be longer, so break them into smaller chunks for
             # the zone file. Consumers are responsible for concatenating the chunks.
             for segment in [pubkey[i:i + 200] for i in range(0, len(pubkey), 200)]:
                 F.write("\"%s\"\n" % (segment))
             F.write(")\n")
      except Exception as e:
         F.write("; Errors:\n")
         for line in str(e).split("\n"):
            F.write("; %s\n" % line)
         pass


def is_ipv6_addr(i):
   try:
      socket.inet_pton(socket.AF_INET6, i)
   except socket.error:
      return False
   return True


def ExtractDNSInfo(x):
   hostname = GetAttr(x, "hostname")

   TTLprefix = "\t"
   if 'dnsTTL' in x[1]:
      TTLprefix = "%s\t" % (x[1]["dnsTTL"][0])

   DNSInfo = []
   if "ipHostNumber" in x[1]:
      for ip in x[1]["ipHostNumber"]:
         if is_ipv6_addr(ip):
            DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, ip))
         else:
            DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, ip))

   Algorithm = None

   ssh_hostnames = [hostname]
   if "sshfpHostname" in x[1]:
      ssh_hostnames += [h for h in x[1]["sshfpHostname"]]

   if 'sshRSAHostKey' in x[1]:
      for key in x[1]["sshRSAHostKey"]:
         Split = key.split()
         key_prefix = Split[0]
         key = base64.decodestring(Split[1])

         # RFC4255
         # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml
         if key_prefix == 'ssh-rsa':
            Algorithm = 1
         if key_prefix == 'ssh-dss':
            Algorithm = 2
         if key_prefix == 'ssh-ed25519':
            Algorithm = 4
         if Algorithm is None:
            continue
         # and more from the registry
         sshfp_digest_codepoints = [(1, 'sha1'), (2, 'sha256')]

         fingerprints = [(digest_codepoint, hashlib.new(algorithm, key).hexdigest()) for digest_codepoint, algorithm in sshfp_digest_codepoints]
         for h in ssh_hostnames:
            for digest_codepoint, fingerprint in fingerprints:
               DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint))

   if 'architecture' in x[1]:
      Arch = GetAttr(x, "architecture")
      Mach = ""
      if "machine" in x[1]:
         Mach = " " + GetAttr(x, "machine")
      DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian"))

   if "mXRecord" in x[1]:
      for mx in x[1]["mXRecord"]:
         if mx in MX_remap:
            for e in MX_remap[mx]:
               DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e))
         else:
            DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, mx))

   return DNSInfo


# Generate the DNS records
def GenZoneRecords(host_attrs, accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Fetch all the hosts
      for x in host_attrs:
         if "hostname" not in x[1]:
            continue

         if IsDebianHost.match(GetAttr(x, "hostname")) is None:
            continue

         for Line in ExtractDNSInfo(x):
            F.write(Line + "\n")

      GenDKIM(accounts, F)
   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate the BSMTP file
def GenBSMTP(accounts, File, HomePrefix):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Write out the zone file entry for each user
      for a in accounts:
         if 'dnsZoneEntry' not in a:
            continue
         if not a.is_active_user():
            continue

         try:
            for z in a["dnsZoneEntry"]:
               Split = z.lower().split()
               if Split[1].lower() == 'in':
                  for y in range(0, len(Split)):
                     if Split[y] == "$":
                        Split[y] = "\n\t"
                  Line = " ".join(Split) + "\n"

                  Host = Split[0] + DNSZone
                  if BSMTPCheck.match(Line) is not None:
                      F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n" % (
                          Host, a['uid'], HomePrefix, a['uid'], Host))

         except Exception:
            F.write("; Errors\n")
            pass

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


def HostToIP(Host, mapped=True):

   IPAdresses = []

   if "ipHostNumber" in Host[1]:
      for addr in Host[1]["ipHostNumber"]:
         IPAdresses.append(addr)
         if not is_ipv6_addr(addr) and mapped == "True":
            IPAdresses.append("::ffff:" + addr)

   return IPAdresses


# Generate the ssh known hosts file
def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None):
   F = None
   try:
      OldMask = os.umask(0o022)
      F = open(File + ".tmp", "w")
      os.umask(OldMask)

      for x in host_attrs:
         if "hostname" not in x[1] or \
            "sshRSAHostKey" not in x[1]:
            continue
         Host = GetAttr(x, "hostname")
         HostNames = [Host]
         if Host.endswith(HostDomain):
            HostNames.append(Host[:-(len(HostDomain) + 1)])

         # in the purpose field [[host|some other text]] (where some other text is optional)
         # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts
         # file.  But so that we don't have to add everything we link we can add an asterisk
         # and say [[*... to ignore it.  In order to be able to add stuff to ssh without
         # http linking it we also support [[-hostname]] entries.
         for i in x[1].get("purpose", []):
            m = PurposeHostField.match(i)
            if m:
               m = m.group(1)
               # we ignore [[*..]] entries
               if m.startswith('*'):
                  continue
               if m.startswith('-'):
                  m = m[1:]
               if m:
                  HostNames.append(m)
                  if m.endswith(HostDomain):
                     HostNames.append(m[:-(len(HostDomain) + 1)])

         for key in x[1]["sshRSAHostKey"]:
            if mode and mode == 'authorized_keys':
               hosts = HostToIP(x)
               if 'sshdistAuthKeysHost' in x[1]:
                  hosts += x[1]['sshdistAuthKeysHost']
               clientcommand = 'rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s' % Host
               clientcommand = "flock -s %s -c '%s'" % (lockfilename, clientcommand)
               Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), key)
            else:
               Line = "%s %s" % (",".join(HostNames + HostToIP(x, False)), key)
            Line = Sanitize(Line) + "\n"
            F.write(Line)
   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


# Generate the debianhosts file (list of all IP addresses)
def GenHosts(host_attrs, File):
   F = None
   try:
      OldMask = os.umask(0o022)
      F = open(File + ".tmp", "w")
      os.umask(OldMask)

      seen = set()

      for x in host_attrs:

         if IsDebianHost.match(GetAttr(x, "hostname")) is None:
            continue

         if 'ipHostNumber' not in x[1]:
            continue

         addrs = x[1]["ipHostNumber"]
         for addr in addrs:
            if addr not in seen:
               seen.add(addr)
               addr = Sanitize(addr) + "\n"
               F.write(addr)

   # Oops, something unspeakable happened.
   except Exception:
      Die(File, F, None)
      raise
   Done(File, F, None)


def replaceTree(src, dst_basedir):
   bn = os.path.basename(src)
   dst = os.path.join(dst_basedir, bn)
   safe_rmtree(dst)
   shutil.copytree(src, dst)


def GenKeyrings(OutDir):
   for k in Keyrings:
      if os.path.isdir(k):
         replaceTree(k, OutDir)
      else:
         shutil.copy(k, OutDir)


def get_accounts(ldap_conn):
   # Fetch all the users
   passwd_attrs = ldap_conn.search_s(
       BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",
       ["uid", "uidNumber", "gidNumber", "supplementaryGid",
        "gecos", "loginShell", "userPassword", "shadowLastChange",
        "shadowMin", "shadowMax", "shadowWarning", "shadowInactive",
        "shadowExpire", "emailForward", "latitude", "longitude",
        "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",
        "keyFingerPrint", "privateSub", "mailDisableMessage",
        "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",
        "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",
        "mailContentInspectionAction", "webPassword", "rtcPassword",
        "bATVToken", "totpSeed", "mailDefaultOptions", "dkimPubKey"])

   if passwd_attrs is None:
      raise UDEmptyList("No Users")
   accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs)
   accounts.sort(key=lambda x: x['uid'].lower())

   return accounts


def get_hosts(ldap_conn):
   # Fetch all the hosts
   HostAttrs = ldap_conn.search_s(
       HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",
       ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",
        "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture",
        "sshfpHostname"])

   if HostAttrs is None:
      raise UDEmptyList("No Hosts")

   HostAttrs.sort(key=lambda x: GetAttr(x, "hostname").lower())

   return HostAttrs


def make_ldap_conn():
   # Connect to the ldap server
   lc = connectLDAP()
   # for testing purposes it's sometimes useful to pass username/password
   # via the environment
   if 'UD_CREDENTIALS' in os.environ:
      Pass = os.environ['UD_CREDENTIALS'].split()
   else:
      F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r")
      Pass = F.readline().strip().split(" ")
      F.close()
   lc.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1])

   return lc


def setup_group_maps(lc):
   # Fetch all the groups
   group_id_map = {}
   subgroup_map = {}
   attrs = lc.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",
                       ["gid", "gidNumber", "subGroup"])

   # Generate the subgroup_map and group_id_map
   for x in attrs:
      if "accountStatus" in x[1] and x[1]['accountStatus'] == "disabled":
         continue
      if "gidNumber" not in x[1]:
         continue
      group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
      if "subGroup" in x[1]:
         subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])

   global SubGroupMap
   global GroupIDMap
   SubGroupMap = subgroup_map
   GroupIDMap = group_id_map


def generate_all(global_dir, ldap_conn):
   accounts = get_accounts(ldap_conn)
   host_attrs = get_hosts(ldap_conn)

   global_dir += '/'
   # Generate global things
   accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")

   accounts = filter(lambda x: not IsRetired(x), accounts)

   CheckForward(accounts)

   GenMailDisable(accounts, global_dir + "mail-disable")
   GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
   GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')
   GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
   GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
   GenCDB(accounts, global_dir + "default-mail-options.cdb", 'mailDefaultOptions')
   GenDBM(accounts, global_dir + "default-mail-options.db", 'mailDefaultOptions')
   GenPrivate(accounts, global_dir + "debian-private")
   GenSSHKnown(host_attrs, global_dir + "authorized_keys", 'authorized_keys', global_dir + 'ud-generate.lock')
   GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
   GenMailBool(accounts, global_dir + "mail-callout", "mailCallout")
   GenMailList(accounts, global_dir + "mail-rbl", "mailRBL")
   GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
   GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
   GenWebPassword(accounts, global_dir + "web-passwords")
   GenRtcPassword(accounts, global_dir + "rtc-passwords")
   GenTOTPSeed(accounts, global_dir + "users.oath")
   GenKeyrings(global_dir)

   # Compatibility.
   GenForward(accounts, global_dir + "forward-alias")

   GenAllUsers(accounts, global_dir + 'all-accounts.json')
   accounts = filter(lambda a: a not in accounts_disabled, accounts)

   ssh_userkeys = GenSSHShadow(global_dir, accounts)
   GenMarkers(accounts, global_dir + "markers")
   GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
   GenHosts(host_attrs, global_dir + "debianhosts")

   GenDNS(accounts, global_dir + "dns-zone")
   GenZoneRecords(host_attrs, accounts, global_dir + "dns-sshfp")

   setup_group_maps(ldap_conn)

   for host in host_attrs:
      if "hostname" not in host[1]:
         continue
      generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)


def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
   current_host = host[1]['hostname'][0]
   OutDir = global_dir + current_host + '/'
   if not os.path.isdir(OutDir):
      os.mkdir(OutDir)

   # Get the group list and convert any named groups to numerics
   GroupList = {}
   for groupname in AllowedGroupsPreload.strip().split(" "):
      GroupList[groupname] = True
   if 'allowedGroups' in host[1]:
      for groupname in host[1]['allowedGroups']:
         GroupList[groupname] = True
   for groupname in GroupList.keys():
      if groupname in GroupIDMap:
         GroupList[str(GroupIDMap[groupname])] = True

   ExtraList = {}
   if 'exportOptions' in host[1]:
      for extra in host[1]['exportOptions']:
         ExtraList[extra.upper()] = True

   if GroupList != {}:
      accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts)

   DoLink(global_dir, OutDir, "debianhosts")
   DoLink(global_dir, OutDir, "ssh_known_hosts")
   DoLink(global_dir, OutDir, "disabled-accounts")

   sys.stdout.flush()
   if 'NOPASSWD' in ExtraList:
      userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*")
   else:
      userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x")
   sys.stdout.flush()
   grouprevmap = GenGroup(accounts, OutDir + "group", current_host)
   GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host)

   # Now we know who we're allowing on the machine, export
   # the relevant ssh keys
   GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)

   if 'NOPASSWD' not in ExtraList:
      GenShadow(accounts, OutDir + "shadow")

   # Link in global things
   if 'NOMARKERS' not in ExtraList:
      DoLink(global_dir, OutDir, "markers")
   DoLink(global_dir, OutDir, "mail-forward.cdb")
   DoLink(global_dir, OutDir, "mail-forward.db")
   DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb")
   DoLink(global_dir, OutDir, "mail-contentinspectionaction.db")
   DoLink(global_dir, OutDir, "mail-disable")
   DoLink(global_dir, OutDir, "mail-greylist")
   DoLink(global_dir, OutDir, "mail-callout")
   DoLink(global_dir, OutDir, "mail-rbl")
   DoLink(global_dir, OutDir, "mail-rhsbl")
   DoLink(global_dir, OutDir, "mail-whitelist")
   DoLink(global_dir, OutDir, "all-accounts.json")
   DoLink(global_dir, OutDir, "default-mail-options.cdb")
   DoLink(global_dir, OutDir, "default-mail-options.db")
   GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
   GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')
   GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
   GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')

   # Compatibility.
   DoLink(global_dir, OutDir, "forward-alias")

   if 'DNS' in ExtraList:
      DoLink(global_dir, OutDir, "dns-zone")
      DoLink(global_dir, OutDir, "dns-sshfp")

   if 'AUTHKEYS' in ExtraList:
      DoLink(global_dir, OutDir, "authorized_keys")

   if 'BSMTP' in ExtraList:
      GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix)

   if 'PRIVATE' in ExtraList:
      DoLink(global_dir, OutDir, "debian-private")

   if 'GITOLITE' in ExtraList:
      GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host)
   if 'exportOptions' in host[1]:
      for entry in host[1]['exportOptions']:
         v = entry.split('=', 1)
         if v[0] != 'GITOLITE' or len(v) != 2:
            continue
         options = v[1].split(',')
         group = options.pop(0)
         gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
         if 'nohosts' not in options:
            gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
         else:
            gitolite_hosts = []
         command = None
         for opt in options:
            if opt.startswith('sshcmd='):
               command = opt.split('=', 1)[1]
         GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s" % group, sshcommand=command, current_host=current_host)

   if 'WEB-PASSWORDS' in ExtraList:
      DoLink(global_dir, OutDir, "web-passwords")

   if 'RTC-PASSWORDS' in ExtraList:
      DoLink(global_dir, OutDir, "rtc-passwords")

   if 'TOTP' in ExtraList:
      DoLink(global_dir, OutDir, "users.oath")

   if 'KEYRING' in ExtraList:
      for k in Keyrings:
         bn = os.path.basename(k)
         if os.path.isdir(k):
            src = os.path.join(global_dir, bn)
            replaceTree(src, OutDir)
         else:
            DoLink(global_dir, OutDir, bn)
   else:
      for k in Keyrings:
         try:
            bn = os.path.basename(k)
            target = os.path.join(OutDir, bn)
            if os.path.isdir(target):
               safe_rmtree(target)
            else:
               posix.remove(target)
         except Exception:
            pass
   DoLink(global_dir, OutDir, "last_update.trace")


def getLastLDAPChangeTime(lc):
   mods = lc.search_s('cn=log',
                      ldap.SCOPE_ONELEVEL,
                      '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))',
                      ['reqEnd'])

   last = 0

   # Sort the list by reqEnd
   sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0])
   # Take the last element in the array
   last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0]

   return last


def getLastKeyringChangeTime():
   krmod = 0
   for k in Keyrings:
      mt = os.path.getmtime(k)
      if mt > krmod:
         krmod = mt

   return int(krmod)


def getLastBuildTime(gdir):
   cache_last_ldap_mod = 0
   cache_last_unix_mod = 0
   cache_last_run = 0

   try:
      fd = open(os.path.join(gdir, "last_update.trace"), "r")
      cache_last_mod = fd.read().split()
      try:
         cache_last_ldap_mod = cache_last_mod[0]
         cache_last_unix_mod = int(cache_last_mod[1])
         cache_last_run = int(cache_last_mod[2])
      except (IndexError, ValueError):
         pass
      fd.close()
   except IOError as e:
      if e.errno == errno.ENOENT:
         pass
      else:
         raise e

   return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)


def mq_notify(options, message):
   options.section = 'dsa-udgenerate'
   options.config = '/etc/dsa/pubsub.conf'

   config = Config(options)
   conf = {
       'rabbit_userid': config.username,
       'rabbit_password': config.password,
       'rabbit_virtual_host': config.vhost,
       'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
       'use_ssl': False
   }

   msg = {
       'message': message,
       'timestamp': int(time.time())
   }
   conn = None
   try:
      conn = Connection(conf=conf)
      conn.topic_send(config.topic,
                      json.dumps(msg),
                      exchange_name=config.exchange,
                      timeout=5)
   finally:
      if conn:
         conn.close()


def ud_generate():
   parser = optparse.OptionParser()
   parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
                     help="Output directory.")
   parser.add_option("-f", "--force", dest="force", action="store_true",
                     help="Force generation, even if no update to LDAP has happened.")

   (options, args) = parser.parse_args()
   if len(args) > 0:
      parser.print_help()
      sys.exit(1)

   if options.generatedir is not None:
      generate_dir = options.generatedir
   elif 'UD_GENERATEDIR' in os.environ:
      generate_dir = os.environ['UD_GENERATEDIR']
   else:
      generate_dir = GenerateDir

   lockf = os.path.join(generate_dir, 'ud-generate.lock')
   lock = get_lock(lockf)
   if lock is None:
      sys.stderr.write("Could not acquire lock %s.\n" % lockf)
      sys.exit(1)

   lc = make_ldap_conn()

   time_started = int(time.time())
   ldap_last_mod = getLastLDAPChangeTime(lc)
   unix_last_mod = getLastKeyringChangeTime()
   cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir)

   need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)

   fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
   if need_update or options.force:
      msg = 'Update forced' if options.force else 'Update needed'
      generate_all(generate_dir, lc)
      if use_mq:
         mq_notify(options, msg)
      last_run = int(time.time())
   fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
   fd.close()
   sys.exit(0)


if __name__ == "__main__":
   if 'UD_PROFILE' in os.environ:
      import cProfile
      import pstats
      cProfile.run('ud_generate()', "udg_prof")
      p = pstats.Stats('udg_prof')
      # p.sort_stats('time').print_stats()
      p.sort_stats('cumulative').print_stats()
   else:
      ud_generate()

# vim:set et:
# vim:set ts=3:
# vim:set shiftwidth=3:
