#!/usr/bin/env python
# -*- mode: python -*-
# Generates passwd, shadow and group files from the ldap directory.

#   Copyright (c) 2000-2001  Jason Gunthorpe <jgg@debian.org>
#   Copyright (c) 2003-2004  James Troup <troup@debian.org>
#   Copyright (c) 2004-2005,7  Joey Schulze <joey@infodrom.org>
#   Copyright (c) 2001-2007  Ryan Murray <rmurray@debian.org>
#   Copyright (c) 2008,2009,2010,2011 Peter Palfrader <peter@palfrader.org>
#   Copyright (c) 2008 Andreas Barth <aba@not.so.argh.org>
#   Copyright (c) 2008 Mark Hymers <mhy@debian.org>
#   Copyright (c) 2008 Luk Claes <luk@debian.org>
#   Copyright (c) 2008 Thomas Viehmann <tv@beamnet.de>
#   Copyright (c) 2009 Stephen Gran <steve@lobefin.net>
#   Copyright (c) 2010 Helmut Grohne <helmut@subdivi.de>
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

from __future__ import print_function

from dsa_mq.connection import Connection
from dsa_mq.config import Config

import string, re, time, ldap, optparse, sys, os, pwd, posix, socket, base64, hashlib, shutil, errno, tarfile, grp, fcntl, dbm
import subprocess
from userdir_ldap import *
from userdir_exceptions import *
import UDLdap
from xml.etree.ElementTree import Element, SubElement, Comment
from xml.etree import ElementTree
from xml.dom import minidom
try:
   from cStringIO import StringIO
except ImportError:
   from StringIO import StringIO
try:
   import simplejson as json
except ImportError:
   import json
   if '__author__' not in json.__dict__:
      sys.stderr.write("Warning: This is probably the wrong json module.  We want python 2.6's json\n")
      sys.stderr.write("module, or simplejson on python 2.5.  Let's see if/how stuff blows up.\n")

if os.getuid() == 0:
   sys.stderr.write("You should probably not run ud-generate as root.\n")
   sys.exit(1)


#
# GLOBAL STATE
#
GroupIDMap = None
SubGroupMap = None



UUID_FORMAT = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
MAX_UD_AGE = 3600*24

EmailCheck = re.compile("^([^ <>@]+@[^ ,<>@]+)(,\s*([^ <>@]+@[^ ,<>@]+))*$")
BSMTPCheck = re.compile(".*mx 0 (master)\.debian\.org\..*",re.DOTALL)
PurposeHostField = re.compile(r".*\[\[([\*\-]?[a-z0-9.\-]*)(?:\|.*)?\]\]")
IsDebianHost = re.compile(ConfModule.dns_hostmatch)
isSSHFP = re.compile("^\s*IN\s+SSHFP")
if UserDNSDomain:
   DNSZone = "." + UserDNSDomain
else:
   DNSZone = ".debian.net"
DKIMSuffix = "_domainkey.debian.org"
Keyrings = ConfModule.sync_keyrings.split(":")
GitoliteSSHRestrictions = getattr(ConfModule, "gitolitesshrestrictions", None)
GitoliteSSHCommand = getattr(ConfModule, "gitolitesshcommand", None)
GitoliteExportHosts = re.compile(getattr(ConfModule, "gitoliteexporthosts", "."))
MX_remap = json.loads(ConfModule.MX_remap)
use_mq = getattr(ConfModule, "use_mq", True)

rtc_realm = getattr(ConfModule, "rtc_realm", None)
rtc_append = getattr(ConfModule, "rtc_append", None)

def prettify(elem):
   """Return a pretty-printed XML string for the Element.
   """
   rough_string = ElementTree.tostring(elem, 'utf-8')
   reparsed = minidom.parseString(rough_string)
   return reparsed.toprettyxml(indent="  ")

def safe_makedirs(dir):
   try:
      os.makedirs(dir)
   except OSError, e:
      if e.errno == errno.EEXIST:
         pass
      else:
         raise e

def safe_rmtree(dir):
   try:
      shutil.rmtree(dir)
   except OSError, e:
      if e.errno == errno.ENOENT:
         pass
      else:
         raise e

def get_lock(fn, wait=5*60):
   f = open(fn, "w")
   sl = 0.1
   ends = time.time() + wait

   while True:
      success = False
      try:
         fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
         return f
      except IOError:
         pass
      if time.time() >= ends:
         return None
      sl = min(sl*2, 10, ends - time.time())
      time.sleep(sl)
   return None


def Sanitize(Str):
   return Str.translate(string.maketrans("\n\r\t", "$$$"))

def DoLink(From, To, File):
   try:
      posix.remove(To + File)
   except:
      pass
   posix.link(From + File, To + File)

def IsRetired(account):
   """
   Looks for accountStatus in the LDAP record and tries to
   match it against one of the known retired statuses
   """

   status = account['accountStatus']

   line = status.split()
   status = line[0]

   if status == "inactive":
      return True

   elif status == "memorial":
      return True

   elif status == "retiring":
      # We'll give them a few extra days over what we said
      age = 6 * 31 * 24 * 60 * 60
      try:
         return (time.time() - time.mktime(time.strptime(line[1], "%Y-%m-%d"))) > age
      except IndexError:
         return False
      except ValueError:
         return False

   return False

# See if this user is in the group list
def IsInGroup(account, allowed, current_host):
  # See if the primary group is in the list
  if str(account['gidNumber']) in allowed: return True

  # Check the host based ACL
  if account.is_allowed_by_hostacl(current_host): return True

  # See if there are supplementary groups
  if 'supplementaryGid' not in account: return False

  supgroups=[]
  addGroups(supgroups, account['supplementaryGid'], account['uid'], current_host)
  for g in supgroups:
     if g in allowed:
        return True
  return False

def Die(File, F, Fdb):
   if F is not None:
      F.close()
   if Fdb is not None:
      Fdb.close()
   try:
      os.remove(File + ".tmp")
   except:
      pass
   try:
      os.remove(File + ".tdb.tmp")
   except:
      pass

def Done(File, F, Fdb):
   if F is not None:
      F.close()
      os.rename(File + ".tmp", File)
   if Fdb is not None:
      Fdb.close()
      os.rename(File + ".tdb.tmp", File + ".tdb")

# Generate the password list
def GenPasswd(accounts, File, HomePrefix, PwdMarker):
   F = None
   try:
      F = open(File + ".tdb.tmp", "w")

      userlist = {}
      i = 0
      for a in accounts:
         if 'loginShell' not in a:
             continue
         # Do not let people try to buffer overflow some busted passwd parser.
         if len(a['gecos']) > 100 or len(a['loginShell']) > 50: continue

         userlist[a['uid']] = a['gidNumber']
         line = "%s:%s:%d:%d:%s:%s%s:%s" % (
                 a['uid'],
                 PwdMarker,
                 a['uidNumber'],
                 a['gidNumber'],
                 a['gecos'],
                 HomePrefix, a['uid'],
                 a['loginShell'])
         line = Sanitize(line) + "\n"
         F.write("0%u %s" % (i, line))
         F.write(".%s %s" % (a['uid'], line))
         F.write("=%d %s" % (a['uidNumber'], line))
         i = i + 1

   # Oops, something unspeakable happened.
   except:
      Die(File, None, F)
      raise
   Done(File, None, F)

   # Return the list of users so we know which keys to export
   return userlist

def GenAllUsers(accounts, file):
   f = None
   try:
      OldMask = os.umask(0022)
      f = open(file + ".tmp", "w", 0644)
      os.umask(OldMask)

      all = []
      for a in accounts:
         all.append( { 'uid': a['uid'],
                       'uidNumber': a['uidNumber'],
                       'active': a.pw_active() and a.shadow_active() } )
      json.dump(all, f)

   # Oops, something unspeakable happened.
   except:
      Die(file, f, None)
      raise
   Done(file, f, None)

# Generate the shadow list
def GenShadow(accounts, File):
   F = None
   try:
      OldMask = os.umask(0077)
      F = open(File + ".tdb.tmp", "w", 0600)
      os.umask(OldMask)

      i = 0
      for a in accounts:
         # If the account is locked, mark it as such in shadow
         # See Debian Bug #308229 for why we set it to 1 instead of 0
         if not a.pw_active():     ShadowExpire = '1'
         elif 'shadowExpire' in a: ShadowExpire = str(a['shadowExpire'])
         else:                     ShadowExpire = ''

         values = []
         values.append(a['uid'])
         values.append(a.get_password())
         for key in 'shadowLastChange', 'shadowMin', 'shadowMax', 'shadowWarning', 'shadowInactive':
            if key in a: values.append(a[key])
            else:        values.append('')
         values.append(ShadowExpire)
         line = ':'.join(values)+':'
         line = Sanitize(line) + "\n"
         F.write("0%u %s" % (i, line))
         F.write(".%s %s" % (a['uid'], line))
         i = i + 1

   # Oops, something unspeakable happened.
   except:
      Die(File, None, F)
      raise
   Done(File, None, F)

# Generate the sudo passwd file
def GenShadowSudo(accounts, File, untrusted, current_host):
   F = None
   try:
      OldMask = os.umask(0077)
      F = open(File + ".tmp", "w", 0600)
      os.umask(OldMask)

      for a in accounts:
         Pass = '*'
         if 'sudoPassword' in a:
            for entry in a['sudoPassword']:
               Match = re.compile('^('+UUID_FORMAT+') (confirmed:[0-9a-f]{40}|unconfirmed) ([a-z0-9.,*-]+) ([^ ]+)$').match(entry)
               if Match is None:
                  continue
               uuid = Match.group(1)
               status = Match.group(2)
               hosts = Match.group(3)
               cryptedpass = Match.group(4)

               if status != 'confirmed:'+make_passwd_hmac('password-is-confirmed', 'sudo', a['uid'], uuid, hosts, cryptedpass):
                  continue
               for_all = hosts == "*"
               for_this_host = current_host in hosts.split(',')
               if not (for_all or for_this_host):
                  continue
               # ignore * passwords for untrusted hosts, but copy host specific passwords
               if for_all and untrusted:
                  continue
               Pass = cryptedpass
               if for_this_host: # this makes sure we take a per-host entry over the for-all entry
                  break
            if len(Pass) > 50:
               Pass = '*'

         Line = "%s:%s" % (a['uid'], Pass)
         Line = Sanitize(Line) + "\n"
         F.write("%s" % (Line))

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate the sudo passwd file
def GenSSHGitolite(accounts, hosts, File, sshcommand=None, current_host=None):
   F = None
   if sshcommand is None:
      sshcommand = GitoliteSSHCommand
   try:
      OldMask = os.umask(0022)
      F = open(File + ".tmp", "w", 0600)
      os.umask(OldMask)

      if not GitoliteSSHRestrictions is None and GitoliteSSHRestrictions != "":
         for a in accounts:
            if 'sshRSAAuthKey' not in a: continue

            User = a['uid']
            prefix = GitoliteSSHRestrictions
            prefix = prefix.replace('@@COMMAND@@', sshcommand)
            prefix = prefix.replace('@@USER@@', User)
            for I in a["sshRSAAuthKey"]:
               if I.startswith("allowed_hosts=") and ' ' in line:
                  if current_host is None:
                     continue
                  machines, I = I.split('=', 1)[1].split(' ', 1)
                  if current_host not in machines.split(','):
                     continue # skip this key

               if I.startswith('ssh-'):
                  line = "%s %s"%(prefix, I)
               else:
                  continue # do not allow keys with other restrictions that might conflict
               line = Sanitize(line) + "\n"
               F.write(line)

         for dn, attrs in hosts:
            if 'sshRSAHostKey' not in attrs: continue
            hostname = "host-" + attrs['hostname'][0]
            prefix = GitoliteSSHRestrictions
            prefix = prefix.replace('@@COMMAND@@', sshcommand)
            prefix = prefix.replace('@@USER@@', hostname)
            for I in attrs["sshRSAHostKey"]:
               line = "%s %s"%(prefix, I)
               line = Sanitize(line) + "\n"
               F.write(line)

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate the shadow list
def GenSSHShadow(global_dir, accounts):
   # Fetch all the users
   userkeys = {}

   for a in accounts:
      if 'sshRSAAuthKey' not in a: continue

      contents = []
      for I in a['sshRSAAuthKey']:
         MultipleLine = "%s" % I
         MultipleLine = Sanitize(MultipleLine)
         contents.append(MultipleLine)
      userkeys[a['uid']] = contents
   return userkeys

# Generate the webPassword list
def GenWebPassword(accounts, File):
   F = None
   try:
      OldMask = os.umask(0077)
      F = open(File, "w", 0600)
      os.umask(OldMask)

      for a in accounts:
         if 'webPassword' not in a: continue
         if not a.pw_active(): continue

         Pass = str(a['webPassword'])
         Line = "%s:%s" % (a['uid'], Pass)
         Line = Sanitize(Line) + "\n"
         F.write("%s" % (Line))

   except:
      Die(File, None, F)
      raise

# Generate the rtcPassword list
def GenRtcPassword(accounts, File):
   F = None
   try:
      OldMask = os.umask(0077)
      F = open(File, "w", 0600)
      os.umask(OldMask)

      for a in accounts:
         if a.is_guest_account(): continue
         if 'rtcPassword' not in a: continue
         if not a.pw_active(): continue

         Line = "%s%s:%s:%s:AUTHORIZED" % (a['uid'], rtc_append, str(a['rtcPassword']), rtc_realm)
         Line = Sanitize(Line) + "\n"
         F.write("%s" % (Line))

   except:
      Die(File, None, F)
      raise

# Generate the TOTP auth file
def GenTOTPSeed(accounts, File):
   F = None
   try:
      OldMask = os.umask(0077)
      F = open(File, "w", 0600)
      os.umask(OldMask)

      F.write("# Option User Prefix Seed\n")
      for a in accounts:
         if a.is_guest_account(): continue
         if 'totpSeed' not in a: continue
         if not a.pw_active(): continue

         Line = "HOTP/T30/6 %s - %s" % (a['uid'], a['totpSeed'])
         Line = Sanitize(Line) + "\n"
         F.write("%s" % (Line))
   except:
      Die(File, None, F)
      raise


def GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, target, current_host):
   OldMask = os.umask(0077)
   tf = tarfile.open(name=os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), mode='w:gz')
   os.umask(OldMask)
   for f in userlist:
      if f not in ssh_userkeys:
         continue
      # If we're not exporting their primary group, don't export
      # the key and warn
      grname = None
      if userlist[f] in grouprevmap.keys():
         grname = grouprevmap[userlist[f]]
      else:
         try:
            if int(userlist[f]) <= 100:
               # In these cases, look it up in the normal way so we
               # deal with cases where, for instance, users are in group
               # users as their primary group.
               grname = grp.getgrgid(userlist[f])[0]
         except Exception, e:
            pass

      if grname is None:
         print("User %s is supposed to have their key exported to host %s but their primary group (gid: %d) isn't in LDAP" % (f, current_host, userlist[f]))
         continue

      lines = []
      for line in ssh_userkeys[f]:
         if line.startswith("allowed_hosts=") and ' ' in line:
            machines, line = line.split('=', 1)[1].split(' ', 1)
            if current_host not in machines.split(','):
               continue # skip this key
         lines.append(line)
      if not lines:
         continue # no keys for this host
      contents = "\n".join(lines) + "\n"

      to = tarfile.TarInfo(name=f)
      # These will only be used where the username doesn't
      # exist on the target system for some reason; hence,
      # in those cases, the safest thing is for the file to
      # be owned by root but group nobody.  This deals with
      # the bloody obscure case where the group fails to exist
      # whilst the user does (in which case we want to avoid
      # ending up with a file which is owned user:root to avoid
      # a fairly obvious attack vector)
      to.uid = 0
      to.gid = 65534
      # Using the username / groupname fields avoids any need
      # to give a shit^W^W^Wcare about the UIDoffset stuff.
      to.uname = f
      to.gname = grname
      to.mode  = 0400
      to.mtime = int(time.time())
      to.size = len(contents)

      tf.addfile(to, StringIO(contents))

   tf.close()
   os.rename(os.path.join(global_dir, 'ssh-keys-%s.tar.gz' % current_host), target)

# add a list of groups to existing groups,
# including all subgroups thereof, recursively.
# basically this proceduces the transitive hull of the groups in
# addgroups.
def addGroups(existingGroups, newGroups, uid, current_host):
   for group in newGroups:
      # if it's a <group>@host, split it and verify it's on the current host.
      s = group.split('@', 1)
      if len(s) == 2 and s[1] != current_host:
         continue
      group = s[0]

      # let's see if we handled this group already
      if group in existingGroups:
         continue

      if not GroupIDMap.has_key(group):
         print("Group", group, "does not exist but", uid, "is in it")
         continue

      existingGroups.append(group)

      if SubGroupMap.has_key(group):
         addGroups(existingGroups, SubGroupMap[group], uid, current_host)

# Generate the group list
def GenGroup(accounts, File, current_host):
   grouprevmap = {}
   F = None
   try:
      F = open(File + ".tdb.tmp", "w")

      # Generate the GroupMap
      GroupMap = {}
      for x in GroupIDMap:
         GroupMap[x] = []
      GroupHasPrimaryMembers = {}

      # Sort them into a list of groups having a set of users
      for a in accounts:
         GroupHasPrimaryMembers[ a['gidNumber'] ] = True
         if 'supplementaryGid' not in a: continue

         supgroups=[]
         addGroups(supgroups, a['supplementaryGid'], a['uid'], current_host)
         for g in supgroups:
            GroupMap[g].append(a['uid'])

      # Output the group file.
      J = 0
      for x in GroupMap.keys():
         if x not in GroupIDMap:
            continue

         if len(GroupMap[x]) == 0 and GroupIDMap[x] not in GroupHasPrimaryMembers:
            continue

         grouprevmap[GroupIDMap[x]] = x

         Line = "%s:x:%u:" % (x, GroupIDMap[x])
         Comma = ''
         for I in GroupMap[x]:
            Line = Line + ("%s%s" % (Comma, I))
            Comma = ','
         Line = Sanitize(Line) + "\n"
         F.write("0%u %s" % (J, Line))
         F.write(".%s %s" % (x, Line))
         F.write("=%u %s" % (GroupIDMap[x], Line))
         J = J + 1

   # Oops, something unspeakable happened.
   except:
      Die(File, None, F)
      raise
   Done(File, None, F)

   return grouprevmap

def CheckForward(accounts):
   for a in accounts:
      if 'emailForward' not in a: continue

      delete = False

      # Do not allow people to try to buffer overflow busted parsers
      if len(a['emailForward']) > 200: delete = True
      # Check the forwarding address
      elif EmailCheck.match(a['emailForward']) is None: delete = True

      if delete:
         a.delete_mailforward()

# Generate the email forwarding list
def GenForward(accounts, File):
   F = None
   try:
      OldMask = os.umask(0022)
      F = open(File + ".tmp", "w", 0644)
      os.umask(OldMask)

      for a in accounts:
         if 'emailForward' not in a: continue
         Line = "%s: %s" % (a['uid'], a['emailForward'])
         Line = Sanitize(Line) + "\n"
         F.write(Line)

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

def GenCDB(accounts, File, key):
   prefix = ["/usr/bin/eatmydata"] if os.path.exists('/usr/bin/eatmydata') else []
   # nothing else does the fsync stuff, so why do it here?
   Fdb = subprocess.Popen(prefix + ["cdbmake", File, "%s.tmp" % File],
                          preexec_fn=lambda: os.umask(0022),
                          stdin=subprocess.PIPE)
   try:
      # Write out the email address for each user
      for a in accounts:
         if key not in a: continue
         value = a[key]
         user = a['uid']
         Fdb.stdin.write("+%d,%d:%s->%s\n" % (len(user), len(value), user, value))

      Fdb.stdin.write("\n")
   finally:
      Fdb.stdin.close()
      if Fdb.wait() != 0:
         raise Exception("cdbmake gave an error")

def GenDBM(accounts, File, key):
   Fdb = None
   OldMask = os.umask(0022)
   fn = os.path.join(File).encode('ascii', 'ignore')
   try:
      posix.remove(fn)
   except:
      pass

   try:
      Fdb = dbm.open(fn, "c")
      os.umask(OldMask)

      # Write out the email address for each user
      for a in accounts:
         if key not in a: continue
         value = a[key]
         user = a['uid']
         Fdb[user] = value

      Fdb.close()
   except:
      # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
      os.remove(File + ".db")
      raise
   # python-dbm names the files Fdb.db.db so we want to them to be Fdb.db
   os.rename (File + ".db", File)

# Generate the anon XEarth marker file
def GenMarkers(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Write out the position for each user
      for a in accounts:
         if not ('latitude' in a and 'longitude' in a): continue
         try:
            Line = "%8s %8s \"\""%(a.latitude_dec(True), a.longitude_dec(True))
            Line = Sanitize(Line) + "\n"
            F.write(Line)
         except:
            pass

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate the debian-private subscription list
def GenPrivate(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Write out the position for each user
      for a in accounts:
         if not a.is_active_user(): continue
         if a.is_guest_account(): continue
         if 'privateSub' not in a: continue
         try:
            Line = "%s"%(a['privateSub'])
            Line = Sanitize(Line) + "\n"
            F.write(Line)
         except:
            pass

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate a list of locked accounts
def GenDisabledAccounts(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")
      disabled_accounts = []

      # Fetch all the users
      for a in accounts:
         if a.pw_active(): continue
         Line = "%s:%s" % (a['uid'], "Account is locked")
         disabled_accounts.append(a)
         F.write(Sanitize(Line) + "\n")

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)
   return disabled_accounts

# Generate the list of local addresses that refuse all mail
def GenMailDisable(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      for a in accounts:
         if 'mailDisableMessage' not in a: continue
         Line = "%s: %s"%(a['uid'], a['mailDisableMessage'])
         Line = Sanitize(Line) + "\n"
         F.write(Line)

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate a list of uids that should have boolean affects applied
def GenMailBool(accounts, File, key):
   F = None
   try:
      F = open(File + ".tmp", "w")

      for a in accounts:
         if key not in a: continue
         if not a[key] == 'TRUE': continue
         Line = "%s"%(a['uid'])
         Line = Sanitize(Line) + "\n"
         F.write(Line)

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate a list of hosts for RBL or whitelist purposes.
def GenMailList(accounts, File, key):
   F = None
   try:
      F = open(File + ".tmp", "w")

      if key == "mailWhitelist": validregex = re.compile('^[-\w.]+(/[\d]+)?$')
      else:                      validregex = re.compile('^[-\w.]+$')

      for a in accounts:
         if key not in a: continue

         filtered = filter(lambda z: validregex.match(z), a[key])
         if len(filtered) == 0: continue
         if key == "mailRHSBL": filtered = map(lambda z: z+"/$sender_address_domain", filtered)
         line = a['uid'] + ': ' + ' : '.join(filtered)
         line = Sanitize(line) + "\n"
         F.write(line)

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

def isRoleAccount(account):
   return 'debianRoleAccount' in account['objectClass']

# Generate the DNS Zone file
def GenDNS(accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Fetch all the users
      RRs = {}

      # Write out the zone file entry for each user
      for a in accounts:
         if 'dnsZoneEntry' not in a: continue
         if not a.is_active_user() and not isRoleAccount(a): continue
         if a.is_guest_account(): continue

         try:
            F.write("; %s\n"%(a.email_address()))
            for z in a["dnsZoneEntry"]:
               Split = z.lower().split()
               if Split[1].lower() == 'in':
                  if Split[2].lower() == 'txt':
                     # TXT records likely have embedded whitespace that should
                     # be retained
                     Line = "%s \"%s\"\n" % (" ".join(Split[0:3]), " ".join(Split[3:]))
                  else:
                     Line = " ".join(Split) + "\n"
                  F.write(Line)

                  Host = Split[0] + DNSZone
                  if BSMTPCheck.match(Line) is not None:
                     F.write("; Has BSMTP\n")

                  # Write some identification information
                  if not RRs.has_key(Host):
                     if Split[2].lower() in ["a", "aaaa"]:
                        Line = "%s IN TXT \"%s\"\n"%(Split[0], a.email_address())
                        for y in a["keyFingerPrint"]:
                           Line = Line + "%s IN TXT \"PGP %s\"\n"%(Split[0], FormatPGPKey(y))
                           F.write(Line)
                        RRs[Host] = 1
               else:
                  Line = "; Err %s"%(str(Split))
                  F.write(Line)

            F.write("\n")
         except Exception, e:
            F.write("; Errors:\n")
            for line in str(e).split("\n"):
               F.write("; %s\n"%(line))
            pass

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate the DKIM records, appending to an open file handle
def GenDKIM(accounts, F):
   try:
      F.write('\n; DKIM\n\n')

      ok_attrs = ['h', 'k']

      # Write out the zone file entry for each user
      for a in accounts:
         if 'dkimPubKey' not in a: continue
         if not a.is_active_user() and not isRoleAccount(a): continue
         if a.is_guest_account(): continue

         try:
            for z in a["dkimPubKey"]:
                # selector, pubkey
                Split = z.split()
                keyattrs = {'h': 'sha256', 'k': 'rsa'}
                selector = Split[0]
                if len(Split) == 3 and Split[1] == 'CNAME':
                   F.write("%s.%s.\tIN\tCNAME\t%s\n" % (selector, DKIMSuffix, Split[2]))
                   continue
                if len(Split) == 2:
                   pubkey = Split[1]
                else:
                   for attr in Split[1:]:
                      k,v = attr.split('=', 1)
                      if v.endswith(';'):
                         v = v[:-1]
                      if k in ok_attrs:
                         keyattrs[k] = v
                      elif k == 'p':
                         pubkey = v
                if pubkey.startswith('p='):
                   pubkey = pubkey[2:]
                Line = "%s.%s.\tIN\tTXT (\"v=DKIM1; k=%s; s=email; h=%s; p=\"\n" % (selector, DKIMSuffix, keyattrs['k'], keyattrs['h'])
                F.write(Line)
                # A single string in a TXT RR cannot be longer than 255 characters.
                # DKIM keys will routinely be longer, so break them into smaller chunks for
                # the zone file. Consumers are responsible for concatenating the chunks.
                for segment in [pubkey[i:i+200] for i in range(0, len(pubkey), 200)]:
                    F.write("\"%s\"\n" % (segment))
                F.write(")\n")
         except Exception, e:
            F.write("; Errors:\n")
            for line in str(e).split("\n"):
               F.write("; %s\n"%(line))
            pass

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise

def is_ipv6_addr(i):
   try:
      socket.inet_pton(socket.AF_INET6, i)
   except socket.error:
      return False
   return True

def ExtractDNSInfo(x):
   hostname = GetAttr(x, "hostname")

   TTLprefix="\t"
   if 'dnsTTL' in x[1]:
      TTLprefix="%s\t"%(x[1]["dnsTTL"][0])

   DNSInfo = []
   if x[1].has_key("ipHostNumber"):
      for I in x[1]["ipHostNumber"]:
         if is_ipv6_addr(I):
            DNSInfo.append("%s.\t%sIN\tAAAA\t%s" % (hostname, TTLprefix, I))
         else:
            DNSInfo.append("%s.\t%sIN\tA\t%s" % (hostname, TTLprefix, I))

   Algorithm = None

   ssh_hostnames = [ hostname ]
   if x[1].has_key("sshfpHostname"):
      ssh_hostnames += [ h for h in x[1]["sshfpHostname"] ]

   if 'sshRSAHostKey' in x[1]:
      for I in x[1]["sshRSAHostKey"]:
         Split = I.split()
         key_prefix = Split[0]
         key = base64.decodestring(Split[1])

         # RFC4255
         # https://www.iana.org/assignments/dns-sshfp-rr-parameters/dns-sshfp-rr-parameters.xhtml
         if key_prefix == 'ssh-rsa':
            Algorithm = 1
         if key_prefix == 'ssh-dss':
            Algorithm = 2
         if key_prefix == 'ssh-ed25519':
            Algorithm = 4
         if Algorithm is None:
            continue
         # and more from the registry
         sshfp_digest_codepoints = [ (1, 'sha1'), (2, 'sha256') ]

         fingerprints = [ ( digest_codepoint, hashlib.new(algorithm, key).hexdigest() ) for digest_codepoint, algorithm in sshfp_digest_codepoints ]
         for h in ssh_hostnames:
            for digest_codepoint, fingerprint in fingerprints:
               DNSInfo.append("%s.\t%sIN\tSSHFP\t%u %d %s" % (h, TTLprefix, Algorithm, digest_codepoint, fingerprint))

   if 'architecture' in x[1]:
      Arch = GetAttr(x, "architecture")
      Mach = ""
      if x[1].has_key("machine"):
         Mach = " " + GetAttr(x, "machine")
      DNSInfo.append("%s.\t%sIN\tHINFO\t\"%s%s\" \"%s\"" % (hostname, TTLprefix, Arch, Mach, "Debian"))

   if x[1].has_key("mXRecord"):
      for I in x[1]["mXRecord"]:
         if I in MX_remap:
            for e in MX_remap[I]:
               DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, e))
         else:
            DNSInfo.append("%s.\t%sIN\tMX\t%s" % (hostname, TTLprefix, I))

   return DNSInfo

# Generate the DNS records
def GenZoneRecords(host_attrs, accounts, File):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Fetch all the hosts
      for x in host_attrs:
         if x[1].has_key("hostname") == 0:
            continue

         if IsDebianHost.match(GetAttr(x, "hostname")) is None:
            continue

         for Line in ExtractDNSInfo(x):
            F.write(Line + "\n")

      GenDKIM(accounts, F)
   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate the BSMTP file
def GenBSMTP(accounts, File, HomePrefix):
   F = None
   try:
      F = open(File + ".tmp", "w")

      # Write out the zone file entry for each user
      for a in accounts:
         if 'dnsZoneEntry' not in a: continue
         if not a.is_active_user(): continue

         try:
            for z in a["dnsZoneEntry"]:
               Split = z.lower().split()
               if Split[1].lower() == 'in':
                  for y in range(0, len(Split)):
                     if Split[y] == "$":
                        Split[y] = "\n\t"
                  Line = " ".join(Split) + "\n"

                  Host = Split[0] + DNSZone
                  if BSMTPCheck.match(Line) is not None:
                      F.write("%s: user=%s group=Debian file=%s%s/bsmtp/%s\n"%(Host,
                                  a['uid'], HomePrefix, a['uid'], Host))

         except:
            F.write("; Errors\n")
            pass

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

def HostToIP(Host, mapped=True):

   IPAdresses = []

   if Host[1].has_key("ipHostNumber"):
      for addr in Host[1]["ipHostNumber"]:
         IPAdresses.append(addr)
         if not is_ipv6_addr(addr) and mapped == "True":
            IPAdresses.append("::ffff:"+addr)

   return IPAdresses

# Generate the ssh known hosts file
def GenSSHKnown(host_attrs, File, mode=None, lockfilename=None):
   F = None
   try:
      OldMask = os.umask(0022)
      F = open(File + ".tmp", "w", 0644)
      os.umask(OldMask)

      for x in host_attrs:
         if x[1].has_key("hostname") == 0 or \
            x[1].has_key("sshRSAHostKey") == 0:
            continue
         Host = GetAttr(x, "hostname")
         HostNames = [ Host ]
         if Host.endswith(HostDomain):
            HostNames.append(Host[:-(len(HostDomain) + 1)])

         # in the purpose field [[host|some other text]] (where some other text is optional)
         # makes a hyperlink on the web thing. we now also add these hosts to the ssh known_hosts
         # file.  But so that we don't have to add everything we link we can add an asterisk
         # and say [[*... to ignore it.  In order to be able to add stuff to ssh without
         # http linking it we also support [[-hostname]] entries.
         for i in x[1].get("purpose", []):
            m = PurposeHostField.match(i)
            if m:
               m = m.group(1)
               # we ignore [[*..]] entries
               if m.startswith('*'):
                  continue
               if m.startswith('-'):
                  m = m[1:]
               if m:
                  HostNames.append(m)
                  if m.endswith(HostDomain):
                     HostNames.append(m[:-(len(HostDomain) + 1)])

         for I in x[1]["sshRSAHostKey"]:
            if mode and mode == 'authorized_keys':
               hosts = HostToIP(x)
               if 'sshdistAuthKeysHost' in x[1]:
                  hosts += x[1]['sshdistAuthKeysHost']
               clientcommand='rsync --server --sender -pr . /var/cache/userdir-ldap/hosts/%s'%(Host)
               clientcommand="flock -s %s -c '%s'"%(lockfilename, clientcommand)
               Line = 'command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,from="%s" %s' % (clientcommand, ",".join(hosts), I)
            else:
               Line = "%s %s" %(",".join(HostNames + HostToIP(x, False)), I)
            Line = Sanitize(Line) + "\n"
            F.write(Line)
   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

# Generate the debianhosts file (list of all IP addresses)
def GenHosts(host_attrs, File):
   F = None
   try:
      OldMask = os.umask(0022)
      F = open(File + ".tmp", "w", 0644)
      os.umask(OldMask)

      seen = set()

      for x in host_attrs:

         if IsDebianHost.match(GetAttr(x, "hostname")) is None:
            continue

         if 'ipHostNumber' not in x[1]:
            continue

         addrs = x[1]["ipHostNumber"]
         for addr in addrs:
            if addr not in seen:
               seen.add(addr)
               addr = Sanitize(addr) + "\n"
               F.write(addr)

   # Oops, something unspeakable happened.
   except:
      Die(File, F, None)
      raise
   Done(File, F, None)

def replaceTree(src, dst_basedir):
   bn = os.path.basename(src)
   dst = os.path.join(dst_basedir, bn)
   safe_rmtree(dst)
   shutil.copytree(src, dst)

def GenKeyrings(OutDir):
   for k in Keyrings:
      if os.path.isdir(k):
         replaceTree(k, OutDir)
      else:
         shutil.copy(k, OutDir)


def get_accounts(ldap_conn):
   # Fetch all the users
   passwd_attrs = ldap_conn.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "(&(uid=*)(!(uidNumber=0))(objectClass=shadowAccount))",\
                   ["uid", "uidNumber", "gidNumber", "supplementaryGid",\
                    "gecos", "loginShell", "userPassword", "shadowLastChange",\
                    "shadowMin", "shadowMax", "shadowWarning", "shadowInactive",
                    "shadowExpire", "emailForward", "latitude", "longitude",\
                    "allowedHost", "sshRSAAuthKey", "dnsZoneEntry", "cn", "sn",\
                    "keyFingerPrint", "privateSub", "mailDisableMessage",\
                    "mailGreylisting", "mailCallout", "mailRBL", "mailRHSBL",\
                    "mailWhitelist", "sudoPassword", "objectClass", "accountStatus",\
                    "mailContentInspectionAction", "webPassword", "rtcPassword",\
                    "bATVToken", "totpSeed", "mailDefaultOptions", "dkimPubKey"])

   if passwd_attrs is None:
      raise UDEmptyList, "No Users"
   accounts = map(lambda x: UDLdap.Account(x[0], x[1]), passwd_attrs)
   accounts.sort(lambda x,y: cmp(x['uid'].lower(), y['uid'].lower()))

   return accounts

def get_hosts(ldap_conn):
   # Fetch all the hosts
   HostAttrs    = ldap_conn.search_s(HostBaseDn, ldap.SCOPE_ONELEVEL, "objectClass=debianServer",\
                   ["hostname", "sshRSAHostKey", "purpose", "allowedGroups", "exportOptions",\
                    "mXRecord", "ipHostNumber", "dnsTTL", "machine", "architecture",
                    "sshfpHostname"])

   if HostAttrs is None:
      raise UDEmptyList, "No Hosts"

   HostAttrs.sort(lambda x, y: cmp((GetAttr(x, "hostname")).lower(), (GetAttr(y, "hostname")).lower()))

   return HostAttrs


def make_ldap_conn():
   # Connect to the ldap server
   l = connectLDAP()
   # for testing purposes it's sometimes useful to pass username/password
   # via the environment
   if 'UD_CREDENTIALS' in os.environ:
      Pass = os.environ['UD_CREDENTIALS'].split()
   else:
      F = open(PassDir + "/pass-" + pwd.getpwuid(os.getuid())[0], "r")
      Pass = F.readline().strip().split(" ")
      F.close()
   l.simple_bind_s("uid=" + Pass[0] + "," + BaseDn, Pass[1])

   return l



def setup_group_maps(l):
   # Fetch all the groups
   group_id_map = {}
   subgroup_map = {}
   attrs = l.search_s(BaseDn, ldap.SCOPE_ONELEVEL, "gid=*",\
                     ["gid", "gidNumber", "subGroup"])

   # Generate the subgroup_map and group_id_map
   for x in attrs:
      if x[1].has_key("accountStatus") and x[1]['accountStatus'] == "disabled":
         continue
      if x[1].has_key("gidNumber") == 0:
         continue
      group_id_map[x[1]["gid"][0]] = int(x[1]["gidNumber"][0])
      if x[1].has_key("subGroup") != 0:
         subgroup_map.setdefault(x[1]["gid"][0], []).extend(x[1]["subGroup"])

   global SubGroupMap
   global GroupIDMap
   SubGroupMap = subgroup_map
   GroupIDMap = group_id_map

def generate_all(global_dir, ldap_conn):
   accounts = get_accounts(ldap_conn)
   host_attrs = get_hosts(ldap_conn)

   global_dir += '/'
   # Generate global things
   accounts_disabled = GenDisabledAccounts(accounts, global_dir + "disabled-accounts")

   accounts = filter(lambda x: not IsRetired(x), accounts)

   CheckForward(accounts)

   GenMailDisable(accounts, global_dir + "mail-disable")
   GenCDB(accounts, global_dir + "mail-forward.cdb", 'emailForward')
   GenDBM(accounts, global_dir + "mail-forward.db", 'emailForward')
   GenCDB(accounts, global_dir + "mail-contentinspectionaction.cdb", 'mailContentInspectionAction')
   GenDBM(accounts, global_dir + "mail-contentinspectionaction.db", 'mailContentInspectionAction')
   GenCDB(accounts, global_dir + "default-mail-options.cdb", 'mailDefaultOptions')
   GenDBM(accounts, global_dir + "default-mail-options.db", 'mailDefaultOptions')
   GenPrivate(accounts, global_dir + "debian-private")
   GenSSHKnown(host_attrs, global_dir+"authorized_keys", 'authorized_keys', global_dir+'ud-generate.lock')
   GenMailBool(accounts, global_dir + "mail-greylist", "mailGreylisting")
   GenMailBool(accounts, global_dir + "mail-callout", "mailCallout")
   GenMailList(accounts, global_dir + "mail-rbl", "mailRBL")
   GenMailList(accounts, global_dir + "mail-rhsbl", "mailRHSBL")
   GenMailList(accounts, global_dir + "mail-whitelist", "mailWhitelist")
   GenWebPassword(accounts, global_dir + "web-passwords")
   GenRtcPassword(accounts, global_dir + "rtc-passwords")
   GenTOTPSeed(accounts, global_dir + "users.oath")
   GenKeyrings(global_dir)

   # Compatibility.
   GenForward(accounts, global_dir + "forward-alias")

   GenAllUsers(accounts, global_dir + 'all-accounts.json')
   accounts = filter(lambda a: a not in accounts_disabled, accounts)

   ssh_userkeys = GenSSHShadow(global_dir, accounts)
   GenMarkers(accounts, global_dir + "markers")
   GenSSHKnown(host_attrs, global_dir + "ssh_known_hosts")
   GenHosts(host_attrs, global_dir + "debianhosts")

   GenDNS(accounts, global_dir + "dns-zone")
   GenZoneRecords(host_attrs, accounts, global_dir + "dns-sshfp")

   setup_group_maps(ldap_conn)

   for host in host_attrs:
      if "hostname" not in host[1]:
         continue
      generate_host(host, global_dir, accounts, host_attrs, ssh_userkeys)

def generate_host(host, global_dir, all_accounts, all_hosts, ssh_userkeys):
   current_host = host[1]['hostname'][0]
   OutDir = global_dir + current_host + '/'
   if not os.path.isdir(OutDir):
      os.mkdir(OutDir)

   # Get the group list and convert any named groups to numerics
   GroupList = {}
   for groupname in AllowedGroupsPreload.strip().split(" "):
      GroupList[groupname] = True
   if 'allowedGroups' in host[1]:
      for groupname in host[1]['allowedGroups']:
         GroupList[groupname] = True
   for groupname in GroupList.keys():
      if groupname in GroupIDMap:
         GroupList[str(GroupIDMap[groupname])] = True

   ExtraList = {}
   if 'exportOptions' in host[1]:
      for extra in host[1]['exportOptions']:
         ExtraList[extra.upper()] = True

   if GroupList != {}:
      accounts = filter(lambda x: IsInGroup(x, GroupList, current_host), all_accounts)

   DoLink(global_dir, OutDir, "debianhosts")
   DoLink(global_dir, OutDir, "ssh_known_hosts")
   DoLink(global_dir, OutDir, "disabled-accounts")

   sys.stdout.flush()
   if 'NOPASSWD' in ExtraList:
      userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "*")
   else:
      userlist = GenPasswd(accounts, OutDir + "passwd", HomePrefix, "x")
   sys.stdout.flush()
   grouprevmap = GenGroup(accounts, OutDir + "group", current_host)
   GenShadowSudo(accounts, OutDir + "sudo-passwd", ('UNTRUSTED' in ExtraList) or ('NOPASSWD' in ExtraList), current_host)

   # Now we know who we're allowing on the machine, export
   # the relevant ssh keys
   GenSSHtarballs(global_dir, userlist, ssh_userkeys, grouprevmap, os.path.join(OutDir, 'ssh-keys.tar.gz'), current_host)

   if 'NOPASSWD' not in ExtraList:
      GenShadow(accounts, OutDir + "shadow")

   # Link in global things
   if 'NOMARKERS' not in ExtraList:
      DoLink(global_dir, OutDir, "markers")
   DoLink(global_dir, OutDir, "mail-forward.cdb")
   DoLink(global_dir, OutDir, "mail-forward.db")
   DoLink(global_dir, OutDir, "mail-contentinspectionaction.cdb")
   DoLink(global_dir, OutDir, "mail-contentinspectionaction.db")
   DoLink(global_dir, OutDir, "mail-disable")
   DoLink(global_dir, OutDir, "mail-greylist")
   DoLink(global_dir, OutDir, "mail-callout")
   DoLink(global_dir, OutDir, "mail-rbl")
   DoLink(global_dir, OutDir, "mail-rhsbl")
   DoLink(global_dir, OutDir, "mail-whitelist")
   DoLink(global_dir, OutDir, "all-accounts.json")
   DoLink(global_dir, OutDir, "default-mail-options.cdb")
   DoLink(global_dir, OutDir, "default-mail-options.db")
   GenCDB(accounts, OutDir + "user-forward.cdb", 'emailForward')
   GenDBM(accounts, OutDir + "user-forward.db", 'emailForward')
   GenCDB(accounts, OutDir + "batv-tokens.cdb", 'bATVToken')
   GenDBM(accounts, OutDir + "batv-tokens.db", 'bATVToken')

   # Compatibility.
   DoLink(global_dir, OutDir, "forward-alias")

   if 'DNS' in ExtraList:
      DoLink(global_dir, OutDir, "dns-zone")
      DoLink(global_dir, OutDir, "dns-sshfp")

   if 'AUTHKEYS' in ExtraList:
      DoLink(global_dir, OutDir, "authorized_keys")

   if 'BSMTP' in ExtraList:
      GenBSMTP(accounts, OutDir + "bsmtp", HomePrefix)

   if 'PRIVATE' in ExtraList:
      DoLink(global_dir, OutDir, "debian-private")

   if 'GITOLITE' in ExtraList:
      GenSSHGitolite(all_accounts, all_hosts, OutDir + "ssh-gitolite", current_host=current_host)
   if 'exportOptions' in host[1]:
      for entry in host[1]['exportOptions']:
         v = entry.split('=',1)
         if v[0] != 'GITOLITE' or len(v) != 2: continue
         options = v[1].split(',')
         group = options.pop(0)
         gitolite_accounts = filter(lambda x: IsInGroup(x, [group], current_host), all_accounts)
         if 'nohosts' not in options:
            gitolite_hosts = filter(lambda x: GitoliteExportHosts.match(x[1]["hostname"][0]), all_hosts)
         else:
            gitolite_hosts = []
         command = None
         for opt in options:
            if opt.startswith('sshcmd='):
               command = opt.split('=',1)[1]
         GenSSHGitolite(gitolite_accounts, gitolite_hosts, OutDir + "ssh-gitolite-%s"%(group,), sshcommand=command, current_host=current_host)

   if 'WEB-PASSWORDS' in ExtraList:
      DoLink(global_dir, OutDir, "web-passwords")

   if 'RTC-PASSWORDS' in ExtraList:
      DoLink(global_dir, OutDir, "rtc-passwords")

   if 'TOTP' in ExtraList:
      DoLink(global_dir, OutDir, "users.oath")

   if 'KEYRING' in ExtraList:
      for k in Keyrings:
         bn = os.path.basename(k)
         if os.path.isdir(k):
            src = os.path.join(global_dir, bn)
            replaceTree(src, OutDir)
         else:
            DoLink(global_dir, OutDir, bn)
   else:
      for k in Keyrings:
         try:
            bn = os.path.basename(k)
            target = os.path.join(OutDir, bn)
            if os.path.isdir(target):
               safe_rmtree(dst)
            else:
               posix.remove(target)
         except:
            pass
   DoLink(global_dir, OutDir, "last_update.trace")


def getLastLDAPChangeTime(l):
   mods = l.search_s('cn=log',
         ldap.SCOPE_ONELEVEL,
         '(&(&(!(reqMod=activity-from*))(!(reqMod=activity-pgp*)))(|(reqType=add)(reqType=delete)(reqType=modify)(reqType=modrdn)))',
         ['reqEnd'])

   last = 0

   # Sort the list by reqEnd
   sorted_mods = sorted(mods, key=lambda mod: mod[1]['reqEnd'][0].split('.')[0])
   # Take the last element in the array
   last = sorted_mods[-1][1]['reqEnd'][0].split('.')[0]

   return last

def getLastKeyringChangeTime():
   krmod = 0
   for k in Keyrings:
      mt = os.path.getmtime(k)
      if mt > krmod:
         krmod = mt

   return int(krmod)

def getLastBuildTime(gdir):
   cache_last_ldap_mod = 0
   cache_last_unix_mod = 0
   cache_last_run = 0

   try:
      fd = open(os.path.join(gdir, "last_update.trace"), "r")
      cache_last_mod=fd.read().split()
      try:
         cache_last_ldap_mod = cache_last_mod[0]
         cache_last_unix_mod = int(cache_last_mod[1])
         cache_last_run = int(cache_last_mod[2])
      except IndexError, ValueError:
         pass
      fd.close()
   except IOError, e:
      if e.errno == errno.ENOENT:
         pass
      else:
         raise e

   return (cache_last_ldap_mod, cache_last_unix_mod, cache_last_run)

def mq_notify(options, message):
   options.section = 'dsa-udgenerate'
   options.config = '/etc/dsa/pubsub.conf'

   config = Config(options)
   conf = {
      'rabbit_userid': config.username,
      'rabbit_password': config.password,
      'rabbit_virtual_host': config.vhost,
      'rabbit_hosts': ['pubsub02.debian.org', 'pubsub01.debian.org'],
      'use_ssl': False
   }

   msg = {
      'message': message,
      'timestamp': int(time.time())
   }
   conn = None
   try:
      conn = Connection(conf=conf)
      conn.topic_send(config.topic,
            json.dumps(msg),
            exchange_name=config.exchange,
            timeout=5)
   finally:
      if conn:
         conn.close()

def ud_generate():
   parser = optparse.OptionParser()
   parser.add_option("-g", "--generatedir", dest="generatedir", metavar="DIR",
     help="Output directory.")
   parser.add_option("-f", "--force", dest="force", action="store_true",
     help="Force generation, even if no update to LDAP has happened.")

   (options, args) = parser.parse_args()
   if len(args) > 0:
      parser.print_help()
      sys.exit(1)

   if options.generatedir is not None:
      generate_dir = options.generatedir
   elif 'UD_GENERATEDIR' in os.environ:
      generate_dir = os.environ['UD_GENERATEDIR']
   else:
      generate_dir = GenerateDir


   lockf = os.path.join(generate_dir, 'ud-generate.lock')
   lock = get_lock( lockf )
   if lock is None:
      sys.stderr.write("Could not acquire lock %s.\n"%(lockf))
      sys.exit(1)

   l = make_ldap_conn()

   time_started = int(time.time())
   ldap_last_mod = getLastLDAPChangeTime(l)
   unix_last_mod = getLastKeyringChangeTime()
   cache_last_ldap_mod, cache_last_unix_mod, last_run = getLastBuildTime(generate_dir)

   need_update = (ldap_last_mod > cache_last_ldap_mod) or (unix_last_mod > cache_last_unix_mod) or (time_started - last_run > MAX_UD_AGE)

   fd = open(os.path.join(generate_dir, "last_update.trace"), "w")
   if need_update or options.force:
      msg = 'Update forced' if options.force else 'Update needed'
      generate_all(generate_dir, l)
      if use_mq:
         mq_notify(options, msg)
      last_run = int(time.time())
   fd.write("%s\n%s\n%s\n" % (ldap_last_mod, unix_last_mod, last_run))
   fd.close()
   sys.exit(0)


if __name__ == "__main__":
   if 'UD_PROFILE' in os.environ:
      import cProfile
      import pstats
      cProfile.run('ud_generate()', "udg_prof")
      p = pstats.Stats('udg_prof')
      ##p.sort_stats('time').print_stats()
      p.sort_stats('cumulative').print_stats()
   else:
      ud_generate()

# vim:set et:
# vim:set ts=3:
# vim:set shiftwidth=3:
