#!/usr/bin/python

### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU Library General Public License as published by
### the Free Software Foundation; version 2 only
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU Library General Public License for more details.
###
### You should have received a copy of the GNU Library General Public License
### along with this program; if not, write to the Free Software
### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
### Copyright 2004-2007 Dag Wieers <dag@wieers.com>

from __future__ import generators # for Python 2.2
import os, sys, glob, re, shutil, getopt, popen2
import ConfigParser, urlparse, sha, types, traceback
import time

__version__ = "$Revision$"
# $Source$

VERSION = '0.8.7'

archs = {
    'alpha': ('alpha', 'alphaev5', 'alphaev56', 'alphaev6', 'alphaev67'),
    'i386': ('i386', 'i486', 'i586', 'i686', 'athlon'),
    'ia64': ('i386', 'i686', 'ia64'),
    'ppc': ('ppc', ),
    'ppc64': ('ppc', 'ppc64', 'ppc64pseries', 'ppc64iseries'),
    'x86_64': ('i386', 'i486', 'i586', 'i686', 'athlon', 'x86_64', 'amd64', 'ia32e'),
    'sparc64': ('sparc', 'sparcv8', 'sparcv9', 'sparc64'),
    'sparc64v': ('sparc', 'sparcv8', 'sparcv9', 'sparcv9v', 'sparc64', 'sparc64v'),
    's390': ('s390', ),
    's390x': ('s390', 's390x'),
}

variables = {}

enable = ('yes', 'on', 'true', '1')
disable = ('no', 'off', 'false', '0')

### Register rhn and rhns as a known schemes
for scheme in ('rhn', 'rhns', 'you'):
    urlparse.uses_netloc.insert(0, scheme)
    urlparse.uses_query.insert(0, scheme)

class Options:
    def __init__(self, args):
        self.configfile = '/etc/mrepo.conf'
        self.dists = []
        self.rhnrelease = None
        self.force = False
        self.dryrun = False
        self.generate = False
        self.quiet = False
        self.remount = False
        self.repos = []
        self.types = []
        self.umount = False
        self.update = False
        self.verbose = 1

        try:
            opts, args = getopt.getopt (args, 'c:d:fghnqr:t:uvx',
                ('config=', 'dist=', 'dryrun', 'force', 'generate', 'help', 'quiet', 'repo',
                'remount', 'type=', 'umount', 'unmount', 'update', 'verbose', 'version', 'extras'))
        except getopt.error, exc:
            print 'mrepo: %s, try mrepo -h for a list of all the options' % str(exc)
            sys.exit(1)

        for opt, arg in opts:
            if opt in ('-c', '--config'):
                self.configfile = os.path.abspath(arg)
            elif opt in ('-d', '--dist'):
                print 'mrepo: the use of -d or --dist as an option is deprecated, use the argument list'
                self.dists = self.dists + arg.split(',')
            elif opt in ('-f', '--force'):
                self.force = True
            elif opt in ('-g', '--generate'):
                self.generate = True
            elif opt in ('-h', '--help'):
                self.usage()
                print
                self.help()
                sys.exit(0)
            elif opt in ('-n', '--dry-run'):
                self.dryrun = True
            elif opt in ('-q', '--quiet'):
                self.quiet = True
            elif opt in ('-r', '--repo'):
                self.repos = self.repos + arg.split(',')
            elif opt in ('--remount', ):
                self.remount = True
            elif opt in ('-t', '--type'):
                self.types = self.types + arg.split(',')
            elif opt in ('-u', '--update'):
                self.update = True
            elif opt in ('--umount', '--unmount'):
                self.umount = True
            elif opt in ('-v', '--verbose'):
                self.verbose = self.verbose + 1
            elif opt in ('--version', ):
                self.version()
                sys.exit(0)
            elif opt in ('-x', '--extras'):
                print 'mrepo: the use of -x or --extras is deprecated, use -u and -r instead'
                self.update = True

        if not self.types:
            self.types = ['file', 'fish', 'ftp', 'http', 'https', 'mc', 'rhn', 'rhns', 'rsync', 'sftp', 'mrepo', 'you']

        for arg in args:
            self.dists = self.dists + arg.split(',')

        if self.quiet:
            self.verbose = 0

        if self.verbose >= 3:
            print 'Verbosity set to level %d' % (self.verbose - 1)
            print 'Using configfile %s' % self.configfile

    def version(self):
        print 'mrepo %s' % VERSION
        print 'Written by Dag Wieers <dag@wieers.com>'
        print 'Homepage at http://dag.wieers.com/home-made/mrepo/'
        print
        print 'platform %s/%s' % (os.name, sys.platform)
        print 'python %s' % sys.version
        print
        print 'build revision $Rev$'

    def usage(self):
        print 'usage: mrepo [options] dist1 [dist2-arch ..]'

    def help(self):
        print '''Set up a distribution server from ISO files

mrepo options:
  -c, --config=file       specify alternative configfile
  -f, --force             force repository generation
  -g, --generate          generate mrepo repositories
  -n, --dry-run           show what would have been done
  -q, --quiet             minimal output
  -r, --repo=repo1,repo2  restrict action to specific repositories
      --remount           remount distribution ISOs
  -u, --update            fetch OS updates
  -v, --verbose           increase verbosity
  -vv, -vvv, -vvvv..      increase verbosity more
      --unmount           unmount distribution ISOs
'''

class Config:
    def __init__(self):
        self.read(op.configfile)

        self.cachedir = self.getoption('main', 'cachedir', '/var/cache/mrepo')
        self.lockdir = self.getoption('main', 'lockdir', '/var/cache/mrepo')
        self.confdir = self.getoption('main', 'confdir', '/etc/mrepo.conf.d')
        self.htmldir = self.getoption('main', 'htmldir', '/usr/share/mrepo/html')
        self.pxelinux = self.getoption('main', 'pxelinux', '/usr/lib/syslinux/pxelinux.0')
        self.srcdir = self.getoption('main', 'srcdir', '/var/mrepo')
        self.tftpdir = self.getoption('main', 'tftpdir', '/tftpboot/mrepo')
        self.wwwdir = self.getoption('main', 'wwwdir', '/var/www/mrepo')
        self.logfile = self.getoption('main', 'logfile', '/var/log/mrepo.log')

        self.mailto = self.getoption('main', 'mailto', None)
        self.mailfrom = self.getoption('main', 'mailfrom', 'mrepo@%s' % os.uname()[1])
        self.smtpserver = self.getoption('main', 'smtp-server', 'localhost')

        self.arch = self.getoption('main', 'arch', 'i386')
        self.metadata = self.getoption('main', 'metadata', 'repomd repoview')

        self.shareiso = self.getoption('main', 'shareiso', 'yes') not in disable

        self.quiet = self.getoption('main', 'quiet', 'no') not in disable
        if op.verbose == 1 and self.quiet:
            op.verbose = 0

        self.hardlink = self.getoption('main', 'hardlink', 'no') not in disable

        ### FIXME: See if fuse module is loaded
        self.fuseiso = self.getoption('main', 'fuseiso', 'yes') not in disable
        self.unionfs = self.getoption('main', 'unionfs', 'yes') not in disable

        self.no_proxy = self.getoption('main', 'no_proxy', None)
        self.ftp_proxy = self.getoption('main', 'ftp_proxy', None)
        self.http_proxy = self.getoption('main', 'http_proxy', None)
        self.https_proxy = self.getoption('main', 'https_proxy', None)

        self.cmd = {}
        self.cmd['createrepo'] = self.getoption('main', 'createrepocmd', '/usr/bin/createrepo')
        self.cmd['fuseiso'] = self.getoption('main', 'fuseisocmd', '/usr/bin/fuseiso')
        self.cmd['genbasedir'] = self.getoption('main', 'genbasedircmd', '/usr/bin/genbasedir')
        self.cmd['hardlink'] = self.getoption('main', 'hardlinkcmd', '/usr/sbin/hardlink')
        self.cmd['hardlink++'] = self.getoption('main', 'hardlinkcppcmd', '/usr/bin/hardlink++')
        self.cmd['hardlinkpy'] = self.getoption('main', 'hardlinkpycmd', '/usr/bin/hardlinkpy')
        self.cmd['lftp'] = self.getoption('main', 'lftpcmd', '/usr/bin/lftp')
        self.cmd['mirrordir'] = self.getoption('main', 'mirrordircmd', '/usr/bin/mirrordir')
        self.cmd['mount'] = self.getoption('main', 'mountcmd', '/bin/mount')
        self.cmd['repoview'] = self.getoption('main', 'repoviewcmd', '/usr/bin/repoview')
        self.cmd['rhnget'] = self.getoption('main', 'rhngetcmd', '/usr/bin/rhnget')
        self.cmd['rsync'] = self.getoption('main', 'rsynccmd', '/usr/bin/rsync')
        self.cmd['unionfs'] = self.getoption('main', 'unionfscmd', '/usr/bin/unionfs')
        self.cmd['umount'] = self.getoption('main', 'umountcmd', '/bin/umount')
        self.cmd['youget'] = self.getoption('main', 'yougetcmd', '/usr/bin/youget')
        self.cmd['yumarch'] = self.getoption('main', 'yumarchcmd', '/usr/bin/yum-arch')

        self.createrepooptions = self.getoption('main', 'createrepo-options', '-p')

        self.lftpbwlimit = self.getoption('main', 'lftp-bandwidth-limit', None)
        self.lftpcleanup = self.getoption('main', 'lftp-cleanup', 'yes') not in disable
        self.lftpexcldebug = self.getoption('main', 'lftp-exclude-debug', 'yes') not in disable
        self.lftpexclsrpm = self.getoption('main', 'lftp-exclude-srpm', 'yes') not in disable
        self.lftpoptions = self.getoption('main', 'lftp-options', '')
        self.lftpcommands = self.getoption('main', 'lftp-commands', '')
        self.lftpmirroroptions = self.getoption('main', 'lftp-mirror-options', '-c -P')
        self.lftptimeout = self.getoption('main', 'lftp-timeout', None)

        self.mirrordircleanup = self.getoption('main', 'mirrordir-cleanup', 'yes') not in disable
        self.mirrordirexcldebug = self.getoption('main', 'mirrordir-exclude-debug', 'yes') not in disable
        self.mirrordirexclsrpm = self.getoption('main', 'mirrordir-exclude-srpm', 'yes') not in disable
        self.mirrordiroptions = self.getoption('main', 'mirrordir-options', '')

        self.rhnlogin = self.getoption('main', 'rhnlogin', None)
        self.rhngetoptions = self.getoption('main', 'rhnget-options', '')
        self.rhngetcleanup = self.getoption('main', 'rhnget-cleanup', 'yes') not in disable
        self.rhngetdownloadall = self.getoption('main', 'rhnget-download-all', 'no') not in disable

        self.rsyncbwlimit = self.getoption('main', 'rsync-bandwidth-limit', None)
        self.rsynccleanup = self.getoption('main', 'rsync-cleanup', 'yes') not in disable
        self.rsyncexcldebug = self.getoption('main', 'rsync-exclude-debug', 'yes') not in disable
        self.rsyncexclsrpm = self.getoption('main', 'rsync-exclude-srpm', 'yes') not in disable
        self.rsyncoptions = self.getoption('main', 'rsync-options', '-rtHL --partial')
        self.rsynctimeout = self.getoption('main', 'rsync-timeout', None)

        self.repoviewoptions = self.getoption('main', 'repoview-options', '')

        self.alldists = []
        self.dists = []

        self.update(op.configfile)

    def read(self, configfile):
        self.cfg = ConfigParser.ConfigParser()

        info(4, 'Reading config file %s' % (configfile))

        (s,b,p,q,f,o) = urlparse.urlparse(configfile)
        if s in ('http', 'ftp', 'file'):
            configfh = urllib.urlopen(configfile)
            try:
                self.cfg.readfp(configfh)
            except ConfigParser.MissingSectionHeaderError, e:
                die(6, 'Error accessing URL: %s' % configfile)
        else:
            if os.access(configfile, os.R_OK):
                try:
                    self.cfg.read(configfile)
                except:
                    die(7, 'Syntax error reading file: %s' % configfile)
            else:
                die(6, 'Error accessing file: %s' % configfile)

    def update(self, configfile):
        for section in ('variables', 'vars', 'DEFAULT'):
            if section in self.cfg.sections():
                for option in self.cfg.options(section):
                    variables[option] = self.cfg.get(section, option)

        for section in self.cfg.sections():
            if section in ('main', 'repos', 'variables', 'vars', 'DEFAULT'):
                continue
            else:
                ### Check if section has appended arch
                for arch in archs.keys():
                    if section.endswith('-%s' % arch):
                        archlist = ( arch, )
                        distname = section.split('-%s' % arch)[0]
                        break
                else:
                    archlist = self.getoption(section, 'arch', self.arch).split()
                    distname = section

                ### Add a distribution for each arch
                for arch in archlist:
                    dist = Dist(distname, arch, self)
                    dist.arch = arch
                    dist.metadata = self.metadata.split()
                    dist.enabled = True
                    dist.promoteepoch = True
                    dist.fuseiso = True
                    dist.unionfs = True
                    for option in self.cfg.options(section):
                        if option in ('iso', 'name', 'release', 'repo', 'rhnrelease'):
                            setattr(dist, option, self.cfg.get(section, option))
                        elif option in ('arch', 'dist'):
                            pass
                        elif option in ('disabled',):
                            dist.enabled = self.cfg.get(section, option) in disable
                        elif option in ('fuseiso',):
                            dist.fuseiso = self.cfg.get(section, option) not in disable
                        elif option in ('unionfs',):
                            dist.unionfs = self.cfg.get(section, option) not in disable
                        elif option in ('metadata',):
                            setattr(dist, option, self.cfg.get(section, option).split())
                        elif option in ('promoteepoch',):
                            dist.promoteepoch = self.cfg.get(section, option) not in disable
                        else:
                            dist.repos.append(Repo(option, self.cfg.get(section, option), dist, self))

                    dist.repos.sort(reposort)
                    dist.rewrite()

                    self.alldists.append(dist)

                    if dist.enabled:
                        self.dists.append(dist)
                    else:
                        info(5, '%s: %s is disabled' % (dist.nick, dist.name))

        self.alldists.sort(distsort)
        self.dists.sort(distsort)

    def getoption(self, section, option, var):
        "Get an option from a section from configfile"
        try:
            var = self.cfg.get(section, option)
            info(2, 'Setting option %s in section [%s] to: %s' % (option, section, var))
        except ConfigParser.NoSectionError, e:
            error(5, 'Failed to find section [%s]' % section)
        except ConfigParser.NoOptionError, e:
#           error(4, 'Failed to find option %s in [%s], set to default: %s' % (option, section, var))
            info(5, 'Setting option %s in section [%s] to: %s (default)' % (option, section, var))
        return var

class Dist:
    def __init__(self, dist, arch, cf):
        self.arch = arch
        self.dist = dist
        self.nick = dist + '-' + arch
        if arch == 'none':
            self.nick = dist
        self.name = dist
        self.dir = os.path.join(cf.wwwdir, self.nick)
        self.iso = None
        self.release = None
        self.repos = []
        self.rhnrelease = None
        self.srcdir = cf.srcdir
        self.discs = ()
        self.isos = []
        self.disabled = False

#   def __repr__(self):
#       for key, value in vars(self).iteritems():
#           if isinstance(value, types.StringType):
#               print key, '->', value

    def rewrite(self):
        "Rewrite (string) attributes to replace variables by other (string) attributes"
        varlist = variables
        varlist.update({ 'arch': self.arch, 'nick': self.nick, 'dist': self.dist,
                'release': self.release, 'rhnrelease': self.rhnrelease })
        for key, value in vars(self).iteritems():
            if isinstance(value, types.StringType):
                setattr(self, key, substitute(value, varlist))
        for repo in self.repos:
            varlist['repo'] = repo.name
            repo.url = substitute(repo.url, varlist)

    def findisos(self):
        "Return a list of existing ISO files"
        if not self.iso: return
        if not self.isos:
            for file in self.iso.split(' '):
                file = os.path.basename(file)
                absfile = file
                if not os.path.isabs(file):
                    absfile = os.path.join(cf.srcdir, self.nick, file)
                    info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
                    filelist = glob.glob(absfile)
                if not filelist:
                    absfile = os.path.join(cf.srcdir, self.dist, file)
                    info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
                    filelist = glob.glob(absfile)
                if not filelist:
                    absfile = os.path.join(cf.srcdir, 'iso', file)
                    info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
                    filelist = glob.glob(absfile)
                if not filelist:
                    absfile = os.path.join(cf.srcdir, file)
                    info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
                    filelist = glob.glob(absfile)
                filelist.sort()
                for iso in filelist:
                    if os.path.isfile(iso) and iso not in self.isos:
                        self.isos.append(iso)
        if self.isos:
            info(5, '%s: Found %d ISO files at %s' % (self.nick, len(self.isos), absfile))
            self.repos.append(Repo('os', '', self, cf))
            self.repos.sort(reposort)
        else:
            info(4, '%s: No ISO files found !' % self.nick)

    def listrepos(self, names=None):
        ret = []
        if names:
            return [ repo for repo in self.repos if repo.name in names ]
        else:
            return self.repos

    def genmetadata(self):
        allsrcdirs = []
        pathjoin = os.path.join
        for repo in self.listrepos(op.repos):
            if not repo.lock('generate'):
                continue
            if repo.name in ('os', 'core') and self.isos:
                repo.url = None
                srcdirs = [ pathjoin(self.dir, disc) for disc in self.discs ]
                self.linksync(repo, srcdirs)
                for file in glob.glob(pathjoin(self.dir + '/disc1/*/base/comps.xml')):
                    if not os.path.exists(pathjoin(self.srcdir, self.nick, 'os-comps.xml')):
                        copy(file, pathjoin(self.srcdir, self.nick, 'os-comps.xml'))
                    allsrcdirs.extend(srcdirs)
            else:
                self.linksync(repo)
                allsrcdirs.append(repo.srcdir)

            repo.check()
            repo.createmd()

            ### After generation, write a sha1sum
            repo.writesha1()
            repo.unlock('generate')

        # do not generate md for 'all', just the links
        self.linksync(Repo('all', '', self, cf), allsrcdirs)

    def linksync(self, repo, srcdirs=None):
        if not srcdirs:
            srcdirs = [ repo.srcdir ]
        destdir = repo.wwwdir
        srcfiles = listrpms(srcdirs, relative = destdir)
        # srcfiles = [ (basename, relpath), ... ]
        srcfiles.sort()
        # uniq basenames
        srcfiles = [f for i, f in enumerate(srcfiles) if not i or f[0] != srcfiles[i-1][0]]

        info(5, '%s: Symlink %s packages from %s to %s' % (repo.dist.nick, repo.name, srcdirs, destdir))
        mkdir(destdir)

        destfiles = listrpmlinks(destdir)
        # destfiles is a list of (link_target_base, link_target_dir) tuples
        destfiles.sort()

        pathjoin = os.path.join

        def keyfunc(x):
            # compare the basenames
            return x[0]

        changed = False
        for srcfile, destfile in synciter(srcfiles, destfiles, key = keyfunc):
            if srcfile is None:
                # delete the link
                base, targetdir = destfile
                linkname = pathjoin(destdir, base)
                info(5, 'Remove link: %s' % (linkname,))
                if not op.dryrun:
                    os.unlink(linkname)
                    changed = True
            elif destfile is None:
                base, srcdir = srcfile
                # create a new link
                linkname = pathjoin(destdir, base)
                target = pathjoin(srcdir, base)
                info(5, 'New link: %s -> %s' % (linkname, target))
                if not op.dryrun:
                    os.symlink(target, linkname)
                    changed = True
            else:
                 # same bases
                 base, srcdir = srcfile
                 base2, curtarget = destfile
                 target = pathjoin(srcdir, base)
                 if target != curtarget:
                     info(5, 'Changed link %s: current: %s, should be: %s' % (base, curtarget, target))
                     linkname = pathjoin(destdir, base)
                     if not op.dryrun:
                         os.unlink(linkname)
                         os.symlink(target, linkname)
                         changed = True

        if changed:
            repo.changed = True

    def mount(self):
        "Loopback mount all ISOs"
        discs = []
        mountpoints = []
        discnr = 0
        if cf.shareiso:
            mkdir(os.path.join(self.dir, 'iso'))
        else:
            remove(os.path.join(self.dir, 'iso'))
        regexp = re.compile('.+[_-]CD[0-9]?\..+')
        ### FIXME: See if fuse module is loaded
        if cf.cmd['fuseiso'] and cf.fuseiso and self.fuseiso:
            opts = '-n'
            extra_opts = '-oallow_other'
            mount_cmd = cf.cmd['fuseiso']
        else:
            opts = '-o loop,ro'
            extra_opts = ''
            mount_cmd = cf.cmd['mount']
            if readfile('/selinux/enforce') == '1':
                opts = opts + ',context=system_u:object_r:httpd_sys_content_t'
        for iso in self.isos:
            if cf.shareiso:
                symlink(iso, os.path.join(self.dir, 'iso'))
            discnr = discnr + 1
            discstr = 'disc'
            if regexp.match(iso, 1):
                discstr = 'CD'
            disc = '%s%s' % (discstr, discnr)
            discs.append(disc)
            mount = os.path.join(self.dir, disc)
            if not os.path.isfile(cf.cmd['mount']):
                die(4, 'mount command not %s' % cf.cmd['mount'])
            mount2 = mountpoint(iso) 
            if not mount2:
                if os.path.exists(mount) and not os.path.isdir(mount):
                    os.rename(mount, os.tempnam(os.path.dirname(mount), 'bak-'))
                mkdir(mount)
                if not os.path.ismount(mount):
                    info(2, '%s: Mount ISO %s to %s' % (self.nick, os.path.basename(iso), mount))
                    run('%s %s %s %s %s' % (mount_cmd, opts, iso, mount, extra_opts))
                    mountpoints.append(mount)
            else:
                if mount2 != mount:
#                    if os.path.exists(mount):
#                        remove(mount)
                    info(5, '%s: %s already mounted, symlink ISO to %s' % (self.nick, os.path.basename(iso), mount))
                    symlink(mount2, mount)

        if cf.cmd['unionfs'] and cf.unionfs and self.unionfs:
            ### This will be the name of our filesystem (first column of /etc/mtab)
            unionfs_name = "%s-%s-fuse" % (self.dist, self.arch)

            ### We need to make sure that our directory isn't already mounted (in the case of mrepo -g)
            if not mountpoint(unionfs_name):
                ### Create the 'os' directory for the merged trees.
                unionfs_mountpoint = os.path.join(self.dir, 'os')
                mkdir(unionfs_mountpoint)
                info(2, "%s -o allow_other,fsname=%s %s %s" % (cf.cmd['unionfs'], unionfs_name, ':'.join(mountpoints), unionfs_mountpoint))
                run("%s -o allow_other,fsname=%s %s %s" % (cf.cmd['unionfs'], unionfs_name, ':'.join(mountpoints), unionfs_mountpoint))

        return discs

    def umount(self):
        "Umount all mounted ISOs"
        discnr = 0
        regexp = re.compile('.+[_-]CD[0-9]?\..+')

        ### Remove any unionfs mounted directories first.
        if os.path.ismount(os.path.join(self.dir, 'os')):
            umount_cmd = 'fusermount -u'
            info(2, '%s %s' % (umount_cmd, os.path.join(self.dir, 'os')))
            run('%s %s' % (umount_cmd, os.path.join(self.dir, 'os')))

        for iso in self.isos:
            discnr = discnr + 1
            discstr = 'disc'
            if regexp.match(iso, 1):
                discstr = 'CD'
            mount = os.path.join(self.dir, discstr + str(discnr))
            if not os.path.isfile(cf.cmd['umount']):
                die(5, 'umount command not %s' % cf.cmd['umount'])
            if os.path.ismount(mount):
                if mountpoint(mount):
                    info(2, '%s: Unmount ISO %s from %s' % (self.nick, os.path.basename(iso), mount))
                    run('%s %s' % (cf.cmd['umount'], mount))
                else:
                    info(2, '%s: Unmount ISO %s from %s' % (self.nick, os.path.basename(iso), mount))
                    run('%s %s' % ('fusermount -u', mount))

    def pxe(self):
        "Create PXE boot setup"
        tftpbootdir = os.path.dirname(cf.tftpdir)
        if cf.tftpdir and tftpbootdir and os.path.isdir(cf.tftpdir):
            tftpdir = os.path.join(cf.tftpdir, self.nick)
            mkdir(tftpdir)
            info(1, '%s: Symlink pxe boot files to %s ' % (self.nick, tftpdir))
            mkdir(os.path.join(tftpdir, 'pxelinux.cfg'))

            ### For Red Hat
            for file in glob.glob(self.dir + '/disc1/images/pxeboot/initrd*.img'):
                copy(file, tftpdir)
            for file in glob.glob(self.dir + '/disc1/images/pxeboot/vmlinuz'):
                copy(file, tftpdir)

            if cf.pxelinux:
                copy(cf.pxelinux, tftpdir)

    def html(self):
        "Put html information in repository"
        mkdir(self.dir)
        if not op.dryrun:
            open(os.path.join(self.dir, '.title'), 'w').write(self.name)
        symlink(os.path.join(cf.htmldir, 'HEADER.repo.shtml'), os.path.join(self.dir, 'HEADER.shtml'))
        symlink(os.path.join(cf.htmldir, 'README.repo.shtml'), os.path.join(self.dir, 'README.shtml'))

class Repo:
    def __init__(self, name, url, dist, cf):
        self.name = name
        self.url = url
        self.dist = dist
        self.srcdir = os.path.join(cf.srcdir, dist.nick, self.name)
        self.wwwdir = os.path.join(dist.dir, 'RPMS.' + self.name)

        self.changed = False

        self.oldlist = set()
        self.newlist = set()

    def __repr__(self):
#       return "%s/%s" % (self.dist.nick, self.name)
        return self.name

    def mirror(self):
        "Check URL and pass on to mirror-functions."
        global exitcode

        ### Do not mirror for repository 'all'
        if self.name == 'all':
            return

        ### Make a snapshot of the directory
        self.oldlist = self.rpmlist()
        self.newlist = self.oldlist

        for url in self.url.split():
            try:
                info(2, '%s: Mirror packages from %s to %s' % (self.dist.nick, url, self.srcdir))
                s, l, p, q, f, o = urlparse.urlparse(url)
                if s not in op.types:
                    info(4, 'Ignoring mirror action for type %s' % s)
                    continue
                if s in ('rsync', ):
                    mirrorrsync(url, self.srcdir)
                elif s in ('ftp', ):
                    if cf.cmd['mirrordir']:
                        mirrormirrordir(url, self.srcdir)
                    else:
                        mirrorlftp(url, self.srcdir)
                elif s in ('fish', 'http', 'https', 'sftp'):
                    mirrorlftp(url, self.srcdir)
                elif s in ('file', ''):
                    mirrorfile(url, self.srcdir)
                elif s in ('mrepo', ):
                    mirrormrepo(url, self.srcdir)
                elif s in ('mc', ):
                    mirrormirrordir(url, self.srcdir)
                elif s in ('rhn', 'rhns'):
                    mirrorrhnget(url, self.srcdir, self.dist)
                elif s in ('you', ):
                    mirroryouget(url, self.srcdir, self.dist)
                else:
                    error(2, 'Scheme %s:// not implemented yet (in %s)' % (s, url))
            except mrepoMirrorException, e:
                error(0, 'Mirroring failed for %s with message:\n  %s' % (url, e.value))
                exitcode = 2
        if not self.url:
            ### Create directory in case no URL is given
            mkdir(self.srcdir)

        ### Make a snapshot of the directory
        self.newlist = self.rpmlist()

    def rpmlist(self):
        "Capture a list of packages in the repository"
        filelist = set()

        ### os.walk() is a python 2.4 feature
#       for root, dirs, files in os.walk(self.srcdir):
#           for file in files:
#               if os.path.exists(file) and file.endswith('.rpm'):
#                   size = os.stat(os.path.join(root, file)).st_size
#                   filelist.add( (file, size) )

        ### os.path.walk() goes back further
        def addfile((filelist, ), path, files):
            for file in files:
                if os.path.exists(os.path.join(path, file)) and file.endswith('.rpm'):
                    size = os.stat(os.path.join(path, file)).st_size
                    filelist.add( (file, size) )

        os.path.walk(self.srcdir, addfile, (filelist,))
        return filelist

    def check(self):
        "Return what repositories require an update and write .newsha1sum"
        if not os.path.isdir(self.wwwdir):
            return
        sha1file = os.path.join(self.wwwdir, '.sha1sum')
        remove(sha1file + '.tmp')
        cursha1 = sha1dir(self.wwwdir)
        if op.force:
            pass
        elif os.path.isfile(sha1file):
            oldsha1 = open(sha1file).read()
            if cursha1 != oldsha1:
                info(2, '%s: Repository %s has new packages.' % (self.dist.nick, self.name))
            else:
                info(5, '%s: Repository %s has not changed. Skipping.' % (self.dist.nick, self.name))
                return
        else:
            info(5, '%s: New repository %s detected.' % (self.dist.nick, self.name))
        writesha1(sha1file + '.tmp', cursha1)
        self.changed = True

    def writesha1(self):
        "Verify .newsha1sum and write a .sha1sum file per repository"
        ### FIXME: Repository 'all' got lost when introducing Repo class
        sha1file = os.path.join(self.wwwdir, '.sha1sum')
        if os.path.isfile(sha1file + '.tmp'):
            cursha1 = sha1dir(self.wwwdir)
            tmpsha1 = open(sha1file + '.tmp').read()
            remove(sha1file + '.tmp')
            if cursha1 == tmpsha1:
                writesha1(sha1file, cursha1)
            else:
                info(5, '%s: Checksum is different. expect: %s, got: %s' % (self.dist.nick, cursha1, tmpsha1))
                info(1, '%s: Directory changed during generating %s repo, please generate again.' % (self.dist.nick, self.name))

    def lock(self, action):
        if op.dryrun:
            return True
        lockfile = os.path.join(cf.lockdir, self.dist.nick, action + '-' + self.name + '.lock')
        mkdir(os.path.dirname(lockfile))
        try:
            fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600)
            info(6, '%s: Setting lock %s' % (self.dist.nick, lockfile))
            os.write(fd, '%d' % os.getpid())
            os.close(fd)
            return True
        except:
            if os.path.exists(lockfile):
                pid = open(lockfile).read()
                if os.path.exists('/proc/%s' % pid):
                    error(0, '%s: Found existing lock %s owned by pid %s' % (self.dist.nick, lockfile, pid))
                else:
                    info(6, '%s: Removing stale lock %s' % (self.dist.nick, lockfile))
                    os.unlink(lockfile)
                    self.lock(action)
                    return True
            else:
                error(0, '%s: Lockfile %s does not exist. Cannot lock. Parallel universe ?' % (self.dist.nick, lockfile))
        return False

    def unlock(self, action):
        if op.dryrun:
            return True
        lockfile = os.path.join(cf.lockdir, self.dist.nick, action + '-' + self.name + '.lock')
        info(6, '%s: Removing lock %s' % (self.dist.nick, lockfile))
        if os.path.exists(lockfile):
            pid = open(lockfile).read()
            if pid == '%s' % os.getpid():
                os.unlink(lockfile)
            else:
                error(0, '%s: Existing lock %s found owned by another process with pid %s. This should NOT happen.' % (self.dist.nick, lockfile, pid))
        else:
            error(0, '%s: Lockfile %s does not exist. Cannot unlock. Something fishy here ?' % (self.dist.nick, lockfile))

    def createmd(self):
        metadata = ('apt', 'createrepo', 'repomd', 'repoview', 'yum')
        index = ('repoview',)

        if not self.changed and not op.force:
            return

        try:
            ### Generate repository metadata
            for md in self.dist.metadata:
                if md in ('createrepo', 'repomd'):
                    self.repomd()
                elif md in ('yum',):
                    self.yum()
                elif md in ('apt',):
                    self.apt()
                elif md not in index:
                    error(0, 'The %s metadata is unknown.' % md)

            ### Generate repository index
            for md in self.dist.metadata:
                if md in ('repoview',):
                    self.repoview()
                elif md not in metadata: 
                    error(0, 'The %s index is unknown.' % md)
        except mrepoGenerateException, e:
            error(0, 'Generating repo failed for %s with message:\n  %s' % (self.name, e.value))
            exitcode = 2

    def repomd(self):
        "Create a repomd repository"
        if not cf.cmd['createrepo']:
            raise mrepoGenerateException('Command createrepo is not found. Skipping.')

        ### Find the createrepo version we are using (due to groupfile usage changes)
        createrepo_version = None
        groupfilename = 'RPMS.%s/comps.xml' % self.name
        try:
            sys.path.append("/usr/share/createrepo")
            import genpkgmetadata
            createrepo_version = genpkgmetadata.__version__
            sys.path.remove("/usr/share/createrepo")
            del genpkgmetadata
        except:
            pass

        ### If version < 0.4.6, then use the old createrepo behaviour
        if not createrepo_version:
            error(0, '%s: Version of createrepo could not be found. Assuming newer than 0.4.6.' % self.dist.nick)
        elif vercmp(createrepo_version, '0.4.6') > 0:
            groupfilename = 'comps.xml'

        opts = ' ' + cf.createrepooptions
        if op.force:
            opts = ' --pretty' + opts
        if op.verbose <= 2:
            opts = ' --quiet' + opts
        elif op.verbose >= 4:
            opts = ' -v' + opts
        if not self.dist.promoteepoch:
            opts = opts + ' -n'
        if os.path.isdir(self.wwwdir):
            repoopts = opts
            if cf.cachedir:
                cachedir = os.path.join(cf.cachedir, self.dist.nick, self.name)
                mkdir(cachedir)
                repoopts = repoopts + ' --cachedir "%s"' % cachedir
            if os.path.isdir(os.path.join(self.wwwdir, '.olddata')):
                remove(os.path.join(self.wwwdir, '.olddata'))
            groupfile = os.path.join(cf.srcdir, self.dist.nick, self.name + '-comps.xml')
            if os.path.isfile(groupfile):
                symlink(groupfile, os.path.join(self.wwwdir, 'comps.xml'))
                repoopts = repoopts + ' --groupfile "%s"' % groupfilename
            info(2, '%s: Create repomd repository for %s' % (self.dist.nick, self.name))
            ret = run('%s %s %s' % (cf.cmd['createrepo'], repoopts, self.wwwdir))
            if ret:
                raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['createrepo'], ret)))

    def yum(self):
        "Create a (old-style) yum repository"
        if not cf.cmd['yumarch']:
            return
        opts = ''
        if op.verbose <= 2:
            opts = ' -q' + opts
        elif op.verbose == 4:
            opts = ' -v' + opts
        elif op.verbose >= 5:
            opts = ' -vv' + opts
        if op.dryrun:
            opts = opts + ' -n'
        if os.path.exists(self.wwwdir):
            if os.path.isdir(os.path.join(self.wwwdir, '.oldheaders')):
                remove(os.path.join(self.wwwdir, '.oldheaders'))
            info(2, '%s: Create (old-style) yum repository for %s' % (self.dist.nick, self.name))
            ret = run('%s %s -l %s' % (cf.cmd['yumarch'], opts, self.wwwdir))
            if ret:
                raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['yumarch'], ret)))

    def apt(self):
        "Create an (old-style) apt repository"
        if not cf.cmd['genbasedir']:
            return
        opts = ''
        if op.verbose >= 3:
            opts = ' --progress' + opts

        mkdir(os.path.join(self.dist.dir, 'base'))

        ### Write out /srcdir/nick/base/release
        # TODO: should not be done per repository
        releasefile = os.path.join(self.dist.dir, 'base', 'release')
        if not os.path.exists(releasefile):
            open(releasefile, 'w').write(
                'Origin: %s\n'\
                'Label: %s\n'\
                'Suite: Unknown\n'\
                'Codename: %s\n'\
                'Date: unknown\n'\
                'Architectures: %s\n'\
                'Components: \n'\
                'Description: %s\n'\
                'MD5Sum:\n'\
                % (os.uname()[1], self.dist.name, self.dist.nick, self.dist.arch, self.dist.name))

        ### Write out /srcdir/nick/base/release.repo
        releasefile = os.path.join(self.dist.dir, 'base', 'release.'+ self.name)
        if not os.path.exists(releasefile):
            open(releasefile, 'w').write(
                'Archive: %s\n'\
                'Component: %s\n'\
                'Version: %s\n'\
                'Origin: %s\n'\
                'Label: Repository %s for %s\n'\
                'Architecture: %s\n'\
                'NotAutomatic: false\n'\
                % (self.name, self.name, self.dist.release, os.uname()[1], self.name, self.dist.name, self.dist.arch))

        info(2, '%s: Create (old-style) apt repository for %s' % (self.dist.nick, self.name))
#       if self.newrepos == self.oldrepos:
#           run('%s %s --flat --bloat --bz2only %s' % (cf.cmd['genbasedir'], opts, self.dist.dir))
#       else:
        ret = run('%s %s --flat --bloat --bz2only --partial %s %s' % (cf.cmd['genbasedir'], opts, self.dist.dir, self.name))
        if ret:
            raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['genbasedir'], ret)))

    def repoview(self):
        "Create a repoview index"
        if not self.changed and not op.force:
            return
        if not cf.cmd['repoview']:
            return
        opts = ''
        if op.force:
            opts = ' --force'
        if op.verbose <= 2:
            opts = ' --quiet' + opts
        if os.path.exists(self.wwwdir):
            info(2, '%s: Create Repoview index for %s' % (self.dist.nick, self.name))
            title = '%s repository for %s' % (self.name, self.dist.nick)
            ret = run('%s %s --title="%s" %s' % (cf.cmd['repoview'], opts, title, self.wwwdir))
            if ret:
                raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['repoview'], ret)))
#           url = 'http://mrepo/%s/RPMS.%s/' % (self.dist.nick, self.name)
#           ret = run('%s %s --url="%s" %s' % (cf.cmd['repoview'], opts, url, self.wwwdir))
#           if ret:
#               raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['repoview'], ret)))

class mySet:
    def __init__(self, seq = ()):
        self._dict = dict([(a, True) for a in seq])

    def add(self, input):
        self._dict[input] = True

    def difference(self, other):
        return mySet([k for k in self._dict.keys() if k not in other])

    def __getitem__(self, key):
        return self._dict[key]

    def __iter__(self):
        return self._dict.__iter__()

    def __repr__(self):
        return 'mySet(%r)' % (self._dict.keys(), )

    __str__ = __repr__

    def __len__(self):
        return len(self._dict)

    def __eq__(self, s):
        return self._dict == s._dict

class mrepoMirrorException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class mrepoGenerateException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

def sha1dir(dir):
    "Return sha1sum of a directory"
    files = glob.glob(dir + '/*.rpm')
    files.sort()
    output = ''
    for file in files:
        output = output + os.path.basename(file) + ' ' + str(os.stat(file).st_size) + '\n'
    return sha.new(output).hexdigest()

def writesha1(file, sha1sum=None):
    "Write out sha1sum"
    repodir = os.path.dirname(file)
    if not sha1sum:
        sha1sum = sha1dir(repodir)
    if not op.dryrun:
        open(file, 'w').write(sha1sum)

def error(level, str):
    "Output error message"
    if level <= op.verbose:
        sys.stderr.write('mrepo: %s\n' % str)

def info(level, str):
    "Output info message"
    if level <= op.verbose:
        sys.stdout.write('%s\n' % str)

def die(ret, str):
    "Print error and exit with errorcode"
    error(0, str)
    sys.exit(ret)

def run(str, dryrun=False):
    "Run command, accept user input, and print output when needed."
    str = 'exec ' + str
    if op.verbose <= 2:
        str = str + ' >/dev/null'
    if not op.dryrun or dryrun:
        info(5, 'Execute: %s' % str)
#       os.popen(str, 'w')
        return os.system(str)
    else:
        info(1, 'Not execute: %s' % str)

def readfile(file, len = 0):
    "Return content of a file"
    if not os.path.isfile(file):
        return None
    if len:
        return open(file, 'r').read(len)
    return open(file, 'r').read()

def writefile(file, str):
    if op.dryrun:
        return
    fd = open(file, 'w')
    fd.write(str)
    fd.close()

_subst_sub = re.compile('\$\{?(\w+)\}?').sub

def substitute(string, vars, recursion = 0):
    "Substitute variables from a string"
    if recursion > 10:
        raise RuntimeError, "variable substitution loop"

    def _substrepl(matchobj):
        value = vars.get(matchobj.group(1))
        if value is not None:
            return substitute(value, vars, recursion + 1)
        return matchobj.group(0)

    string = _subst_sub(_substrepl, string)
    return string

def mountpoint(dev):
    "Return the mountpoint of a mounted device/file"
    for entry in readfile('/etc/mtab').split('\n'):
        if entry:
            cols = entry.split()
            if dev == cols[0]:
                return cols[1]

def distsort(a, b):
    return cmp(a.nick, b.nick)

def reposort(a, b):
    return cmp(a.name, b.name)

def vercmp(a, b):
    al = a.split('.')
    bl = b.split('.')
    minlen = min(len(al), len(bl))
    for i in range(1, minlen):
        if cmp(al[i], bl[i]) < 0:
            return -1
        elif cmp(al[i], bl[i]) > 0:
            return 1
    return cmp(len(al), len(bl))

def symlinkglob(str, *targets):
    "Symlink files to multiple targets"
    for file in glob.glob(str):
        for target in targets:
            mkdir(target)
            symlink(file, target)

def abspath(path, reference):
    "Make absolute path from reference"
    return os.path.normpath(os.path.join(path, reference))

def relpath(path, reference):
    """Make relative path from reference
       if reference is a directory, it must end with a /"""
    common = os.path.commonprefix([path, reference])
    common = common[0:common.rfind('/')+1]
    (uncommon, targetName) = os.path.split(reference.replace(common, '', 1))
    if uncommon:
        newpath = []
        for component in uncommon.split('/'):
            newpath.append('..')
        newpath.append(path.replace(common, '', 1))
        return '/'.join(newpath)
    else:
        return path

def symlink(src, dst):
    "Create a symbolic link, force if dst exists"
    if op.dryrun:
        return
    elif os.path.islink(dst):
        if os.path.samefile(src, abspath(os.readlink(dst), src)):
            return
        os.unlink(dst)
    elif os.path.isdir(dst):
        if os.path.isdir(src):
            if os.path.samefile(src, dst):
                return
        else:
            dst = os.path.join(dst, os.path.basename(src))
            symlink(src, dst)
            return
    elif os.path.isfile(dst):
        if os.path.samefile(src, dst):
            return
        os.rename(dst, dst+'.mrepobak')

### Not using filecmp increases speed with 15%
#   if os.path.isfile(dst) and filecmp.cmp(src, dst) == 0:

    src = relpath(src, dst)

    if not os.path.isdir(os.path.dirname(dst)):
        mkdir(os.path.dirname(dst))
    os.symlink(src, dst)

def copy(src, dst):
    "Copy a file, force if dst exists"
    if op.dryrun:
        return
    if os.path.isdir(dst):
        dst = os.path.join(dst, os.path.basename(src))
    if os.path.islink(dst) or os.path.isfile(dst):
        os.unlink(dst)
    mkdir(os.path.dirname(dst))
    if not os.path.exists(dst):
        if os.path.isfile(src):
            shutil.copy2(src, dst)
        elif os.path.isdir(src):
            shutil.copytree(src, dst)

def remove(file):
    "Remove files or directories"
    if isinstance(file, types.StringType):
        if op.dryrun:
            return
        if os.path.islink(file):
            os.unlink(file)
        elif os.path.isdir(file):
            try:
                os.rmdir(file)
            except:
                os.path.walk(file, removedir, ())
                os.rmdir(file)
        elif os.path.isfile(file) or os.path.islink(file):
            os.unlink(file)
    else:
        for f in file:
            remove(f)

def removedir(void, dir, files):
    for file in files:
        remove(os.path.join(dir, file))

def mkdir(path):
    "Create a directory, and parents if needed"
    if op.dryrun:
        return
    if os.path.islink(path):
        os.unlink(path)
    if not os.path.exists(path):
        os.makedirs(path)

def mirrorrsync(url, path):
    "Mirror everything from an rsync:// URL"
    if not cf.cmd['rsync']:
        error(1, 'rsync was not found. rsync support is therefor disabled.')
        return
    mkdir(path)

    opts = cf.rsyncoptions
    if op.verbose <= 2:
        opts = opts + ' -q'
    elif op.verbose == 3:
        opts = opts + ' -v'
    elif op.verbose == 4:
        opts = opts + ' -v --progress'
    elif op.verbose == 5:
        opts = opts + ' -vv --progress'
    elif op.verbose >= 6:
        opts = opts + ' -vvv --progress'
    if op.dryrun:
        opts = opts + ' --dry-run'
    if cf.rsynctimeout:
        opts = opts + ' --timeout=%s' % cf.rsynctimeout
    if cf.rsynccleanup:
        opts = opts + ' --delete-after --delete-excluded'
    if cf.rsyncbwlimit:
        opts = opts + ' --bwlimit=%s' % cf.rsyncbwlimit
    opts = opts + ' --exclude=\"/headers/\" --exclude=\"/repodata/\"'
    if cf.rsyncexclsrpm:
        opts = opts + ' --exclude=\"*.src.rpm\" --exclude=\"/SRPMS/\"'
    if cf.rsyncexcldebug:
        opts = opts + ' --exclude=\"*-debuginfo-*.rpm\" --exclude=\"/debug/\"'
    opts = opts + ' --include=\"*.rpm\"'
    if cf.rsyncexclsrpm or cf.rsyncexcldebug:
        opts = opts + ' --exclude=\"*.*\"'

    ret = run('%s %s %s %s' % (cf.cmd['rsync'], opts, url, path), dryrun=True)
    if ret:
        raise(mrepoMirrorException('Failed with return code: %s' % ret))

def mirrormirrordir(url, path):
    "Mirror everything from a ftp:// or mc:// URL"
    if not cf.cmd['mirrordir']:
        error(1, 'mirrordir was not found. ftp and mc support (using mirrordir) is therefor disabled.')
        return
    mkdir(path)

    opts = cf.mirrordiroptions
    if op.verbose >= 3:
        opts = opts + ' -v' * (op.verbose - 3)  
    if op.dryrun:
        opts = opts + ' --dry-run'
    if cf.mirrordircleanup:
        opts = opts + ' -k'

#   opts = opts + ' -I \"*.rpm\"'
    opts = opts + ' -G \"headers\" -G \"repodata\"'
    if cf.mirrordirexclsrpm:
        opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
    if cf.mirrordirexcldebug:
        opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'

    ret = run("%s %s '%s' '%s'" % (cf.cmd['mirrordir'], opts, url, path), dryrun=True)
    if ret:
        raise(mrepoMirrorException('Failed with return code: %s' % ret))


def mirrorlftp(url, path):
    "Mirror everything from a http://, ftp://, sftp://, fish:// URL"
    if not cf.cmd['lftp']:
        error(1, 'lftp was not found. fish, ftp, http and sftp support (using lftp) is therefor disabled.')
        return
    mkdir(path)

    cmds = cf.lftpcommands + ';'
#   cmds = 'set dns:fatal-timeout 5'
    if cf.lftptimeout:
        cmds = cmds + ' set net:timeout %s;' % cf.lftptimeout
    if cf.lftpbwlimit:
        cmds = cmds + ' set net:limit-total-rate %s:0;' % cf.lftpbwlimit

    opts = cf.lftpoptions
    if op.verbose >= 6:
        opts = opts + ' -d'

    mirroropts = cf.lftpmirroroptions
    if op.verbose >= 3:
        mirroropts = mirroropts + ' -v' * (op.verbose - 2)
    if op.dryrun:
        mirroropts = mirroropts + ' --dry-run'
    if cf.lftpcleanup:
        mirroropts = mirroropts + ' -e'
    mirroropts = mirroropts + ' -I *.rpm -X \"/headers/\" -X \"/repodata/\"'
    if cf.lftpexclsrpm:
        mirroropts = mirroropts + ' -X \"*.src.rpm\" -X \"/SRPMS/\"'
    if cf.lftpexcldebug:
        mirroropts = mirroropts + ' -X \"*-debuginfo-*.rpm\" -X \"/debug/\"'

    ret = run('%s %s -c \'%s mirror %s %s %s\'' % (cf.cmd['lftp'], opts, cmds, mirroropts, url, path), dryrun=True)
    if ret:
        raise(mrepoMirrorException('Failed with return code: %s' % ret))

def mirrorfile(url, path):
    "Mirror everything from a file:// URL by symlinking"
    dir = url.replace('file://', '')
#   while dir.endswith('/'):
#       dir = dir[0:-1]
    if os.path.isdir(dir):
        symlink(dir, path)
#   else: ### FIXME: Only if ISO file
#       if not os.path.isabs(file):
#           file = os.path.join(cf.srcdir, 'iso', file)
#       isolist = glob.glob(file)
#       isolist.sort()
#       for iso in isolist:
#           if os.path.isfile(iso):
#               print 'Please mount %s to %s' % (iso, path)

def mirrormrepo(url, path):
    "Mirror everything from a local mrepo mirror by symlinking"
    pathname = url.replace('mrepo://', '')
    while pathname.endswith('/'):
        pathname = pathname[0:-1]
    while path.endswith('/'):
        path = pathname[0:-1]
    symlink(os.path.join(cf.srcdir, pathname), path)
#   basename = os.path.basename(pathname)
#   symlink(os.path.join(cf.srcdir, pathname), os.path.join(path, basename))

def mirrorrhnget(url, path, dist):
    "Mirror everything from a rhn:// or rhns:// URL"
    if not cf.cmd['rhnget']:
        error(1, 'rhnget was not found. rhn and rhns support is therefor disabled.')
        return
    mkdir(path)

    opts = cf.rhngetoptions
    if op.verbose >= 3:
        opts = opts + ' -v' * (op.verbose - 3)  
    if op.dryrun:
        opts = opts + ' --dry-run'
    if cf.rhngetcleanup:
        opts = opts + ' --delete'
    if cf.rhngetdownloadall:
        opts = opts + ' --download-all'

    systemidpath = os.path.join(cf.srcdir, dist.nick, 'systemid')
    if os.path.isfile(systemidpath):
        opts = opts + ' --systemid="%s"' % systemidpath

    if dist.rhnrelease:
        opts = opts + ' --release="%s"' % dist.rhnrelease

    if cf.rhnlogin:
        rhnlogin = cf.rhnlogin.split(':')
        if len(rhnlogin) > 0:
            opts = opts + ' --username="%s"' % rhnlogin[0]
        if len(rhnlogin) > 1:
            opts = opts + ' --password="%s"' % rhnlogin[1]

##  opts = opts + ' -I \"*.rpm\"'
#   opts = opts + ' -G \"headers\" -G \"repodata\"'
#   if cf.mirrordirexclsrpm:
#       opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
#   if cf.mirrordirexcldebug:
#       opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'

    ret = run("%s %s '%s' '%s'" % (cf.cmd['rhnget'], opts, url, path), dryrun=True)
    if ret:
        raise(mrepoMirrorException('Failed with return code: %s' % ret))

def mirroryouget(url, path, dist):
    "Mirror everything from a you:// URL"
    if not cf.cmd['youget']:
        error(1, 'youget was not found. YOU support is therefor disabled.')
        return
    mkdir(path)

    url = url.replace('you://', 'https://')

    opts = cf.rhngetoptions
    if op.verbose >= 3:
        opts = opts + ' -v' * (op.verbose - 3)  
    if op.dryrun:
        opts = opts + ' --dry-run'
    if cf.rhngetcleanup:
        opts = opts + ' --delete'
    if cf.rhngetdownloadall:
        opts = opts + ' --download-all'

    credpath = os.path.join(cf.srcdir, dist.nick)
    if os.path.isdir(credpath):
        opts = opts + ' --credpath="%s"' % credpath

#    if dist.rhnrelease:
#        opts = opts + ' --release="%s"' % dist.rhnrelease

#    if cf.rhnlogin:
#        rhnlogin = cf.rhnlogin.split(':')
#        if len(rhnlogin) > 0:
#            opts = opts + ' --username="%s"' % rhnlogin[0]
#        if len(rhnlogin) > 1:
#            opts = opts + ' --password="%s"' % rhnlogin[1]

##  opts = opts + ' -I \"*.rpm\"'
#   opts = opts + ' -G \"headers\" -G \"repodata\"'
#   if cf.mirrordirexclsrpm:
#       opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
#   if cf.mirrordirexcldebug:
#       opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'

    ret = run("%s %s '%s' '%s'" % (cf.cmd['youget'], opts, url, path), dryrun=True)
    if ret:
        raise(mrepoMirrorException('Failed with return code: %s' % ret))

def hardlink(srcdir):
    info(1, 'Hardlinking duplicate packages in %s.' % srcdir)
    opts = ''
    if cf.cmd['hardlinkpy']:
        if op.verbose <= 2:
            opts = ' -v 0'
        else:
            opts = ' -v %d' % (op.verbose - 2)
        run('%s %s %s' % (cf.cmd['hardlinkpy'], os.path.join(srcdir, ''), opts))
    elif cf.cmd['hardlink++']:
        if op.verbose <= 2:
            opts = '>/dev/null'
        run('%s %s %s' % (cf.cmd['hardlink++'], os.path.join(srcdir, ''), opts))
    elif cf.cmd['hardlink']:
        if op.verbose:
            opts = opts + ' -' + ('v' * (op.verbose - 2))
        if op.dryrun:
            opts = opts + ' -n'
        run('%s -c %s %s' % (cf.cmd['hardlink'], opts, os.path.join(srcdir, '')), dryrun=True)
    else:
        info(1, 'hardlink was not found, hardlink support is therefor disabled.')
        return

def rpmlink((dist, repo), dirpath, filelist):
    archlist = ['noarch', ]
    if archs.has_key(dist.arch):
        archlist.extend(archs[dist.arch])
    else:
        archlist.append(dist.arch)
    for arch in archlist:
        regexp = re.compile('.+[\._-]' + arch + '\.rpm$')
        for file in filelist:
            src = os.path.join(dirpath, file)
            if os.path.islink(src) and os.path.isdir(src):
                os.path.walk(src, rpmlink, (dist, repo))
            elif regexp.match(file, 1):
                symlink(src, os.path.join(dist.dir, 'RPMS.' + repo))
                symlink(src, os.path.join(dist.dir, 'RPMS.all'))
    
def which(cmd):
    "Find executables in PATH environment"
    for path in os.environ.get('PATH','$PATH').split(':'):
        if os.path.isfile(os.path.join(path, cmd)):
            info(5, 'Found command %s in path %s' % (cmd, path))
            return os.path.join(path, cmd)
    return ''

def htmlindex():
    symlink(cf.htmldir + '/HEADER.index.shtml', cf.wwwdir + '/HEADER.shtml')
    symlink(cf.htmldir + '/README.index.shtml', cf.wwwdir + '/README.shtml')

def mail(subject, msg):
    info(2, 'Sending mail to: %s' % cf.mailto)
    try:
        import smtplib
        smtp = smtplib.SMTP(cf.smtpserver)
#       server.set_debuglevel(1)
        msg = 'Subject: [mrepo] %s\nX-Mailer: mrepo %s\n\n%s' % (subject, VERSION, msg)
        for email in cf.mailto.split():
            smtp.sendmail(cf.mailfrom, email, 'To: %s\n%s' % (email, msg))
        smtp.quit()
    except:
        info(1, 'Sending mail via %s failed.' % cf.smtpserver)

def readconfig():
    cf = Config()
    if cf.confdir and os.path.isdir(cf.confdir):
        files = glob.glob(os.path.join(cf.confdir, '*.conf'))
        files.sort()
        for configfile in files:
            cf.read(configfile)
            cf.update(configfile)
    return cf

def _nextNone(iterator):
    try:
        return iterator.next()
    except StopIteration:
        return None

def synciter(a, b, key = None, keya = None, keyb = None):
    """returns an iterator that compares two ordered iterables a and b.
    If keya or keyb are specified, they are called with elements of the corresponding
    iterable. They should return a value that is used to compare two elements.
    If keya or keyb are not specified, they default to key or to the element itself,
    if key is None."""

    if key is None:
        key = lambda x: x
    if keya is None:
        keya = key
    if keyb is None:
        keyb = key
    ai = iter(a)
    bi = iter(b)
    aelem = _nextNone(ai)
    belem = _nextNone(bi)
    while not ((aelem is None) or (belem is None)):
        akey = keya(aelem)
        bkey = keyb(belem)
        if akey == bkey:
            yield aelem, belem
            aelem = _nextNone(ai)
            belem = _nextNone(bi)
        elif akey > bkey:
            # belem missing in a
            yield None, belem
            belem = _nextNone(bi)
        elif bkey > akey:
            # aelem missing in b
            yield aelem, None
            aelem = _nextNone(ai)
    # rest
    while aelem is not None:
        akey = key(aelem)
        yield aelem, None
        aelem = _nextNone(ai)
    while belem is not None:
        bkey = key(belem)
        yield None, belem
        belem = _nextNone(bi)

def listrpms(dirs, relative = ''):
    """return a list of rpms in the given directories as a list of (name, path) tuples
    if relative is specified, return the paths relative to this directory"""
    if not isinstance(dirs, (list, tuple)):
        dirs = ( dirs, )
    if relative and not relative.endswith('/'):
        relative += '/'
    isdir = os.path.isdir
    pathjoin = os.path.join
    pathexists = os.path.exists

    def processdir(rpms, path, files):
        if relative:
            path2 = relpath(path, relative)
        else:
            path2 = path
        for f in files:
            pf = pathjoin(path, f)
            if f.endswith('.rpm') and pathexists(pf) and not isdir(pf):
                rpms.append((f, path2))

    rpms = []
    for dir in dirs:
        if not dir.startswith('/'):
            dir = pathjoin(relative, dir)
        os.path.walk(dir, processdir, rpms)
    rpms.sort()
    return rpms

def listrpmlinks(dir):
    islink = os.path.islink
    readlink = os.readlink
    pathjoin = os.path.join
    links = []
    for f in os.listdir(dir):
        path = pathjoin(dir, f)
        if islink(path) and f.endswith('.rpm'):
            links.append((f, readlink(path)))
    return links

def main():
    ### Check availability of commands
    for cmd in cf.cmd.keys():
        if not cf.cmd[cmd]:
            continue
        cmdlist = cf.cmd[cmd].split()
        if not os.path.isfile(cmdlist[0]):
            cmdlist[0] = which(cmdlist[0])
        if cmdlist[0] and not os.path.isfile(cmdlist[0]):
            error(4, '%s command not found as %s, support disabled' % (cmd, cmdlist[0]))
            cf.cmd[cmd] = ''
        else:
            cf.cmd[cmd] = ' '.join(cmdlist)
    if not cf.cmd['createrepo'] and not cf.cmd['yumarch'] and not cf.cmd['genbasedir']:
        error(1, 'No tools found to generate repository metadata. Please install apt, yum or createrepo.')

    ### Set proxy-related environment variables
    if cf.no_proxy:
        os.environ['no_proxy'] = cf.no_proxy
    if cf.ftp_proxy:
        os.environ['ftp_proxy'] = cf.ftp_proxy
    if cf.http_proxy:
        os.environ['http_proxy'] = cf.http_proxy
    if cf.https_proxy:
        os.environ['https_proxy'] = cf.https_proxy

    ### Select list of distributions in order of appearance
    if not op.dists:
        dists = cf.dists
    else:
        dists = []
        for name in op.dists:
            append = False
            for dist in cf.alldists:
                if name == dist.nick or name == dist.dist:
                    dists.append(dist)
                    append = True
            if not append:
                error(1, 'Distribution %s not defined' % name)

    sumnew = 0
    sumremoved = 0
    msg = 'The following changes to mrepo\'s repositories on %s have been made:' % os.uname()[1]

    ### Mounting and mirroring available distributions/repositories
    for dist in dists:
        dist.findisos()
        ### Mount ISOs
        if dist.isos:
            if op.umount or op.remount:
                dist.umount()
            if not op.umount or op.remount:
                dist.discs = dist.mount()

        if op.update:
            msg = msg + '\n\nDist: %s (%s)' % (dist.name, dist.nick)
            info(1, '%s: Updating %s' % (dist.nick, dist.name))

            distnew = 0
            distremoved = 0

            ### Downloading things
            for repo in dist.listrepos(op.repos):
                if not repo.lock('update'):
                    continue
                if repo.name in ('os', 'core'):
                    if not dist.isos:
                        repo.mirror()
                elif repo in dist.listrepos():
                    repo.mirror()
                else:
                    info(2, '%s: Repository %s does not exist' % (dist.nick, repo.name))
                    repo.unlock('update')
                    continue

                repo.unlock('update')

                ### files whose size has changed are in new and removed!
                new = repo.newlist.difference(repo.oldlist)
                removed = repo.oldlist.difference(repo.newlist)

                if new or removed:
                    msg = msg + '\n\n\tRepo: %s' % repo.name
                    info(2, '%s: Repository %s changed (new: %d, removed: %d)' % (dist.nick, repo.name, len(new), len(removed)))
                    fd = open(cf.logfile, 'a+')
                    date = time.strftime("%b %d %H:%M:%S", time.gmtime())

                    def sortedlist(pkgs):
                        l = list(pkgs)
                        l.sort()
                        return l

                    def formatlist(pkglist):
                        return '\n\t' + '\n\t'.join([elem[0] for elem in pkglist])

                    if new:
                        pkglist = sortedlist(new)
                        info(4, '%s: New packages: %s' % (dist.nick, formatlist(pkglist)))
                        distnew += len(pkglist)
                        for element in pkglist:
                            fd.write('%s %s/%s Added %s (%d kiB)\n' % (date, dist.nick, repo.name, element[0], element[1]/1024))
                            msg = msg + '\n\t\t+ %s (%d kiB)' % (element[0], element[1]/1024)

                    if removed:
                        pkglist = sortedlist(removed)
                        info(4, '%s: Removed packages: %s' % (dist.nick, formatlist(pkglist)))
                        distremoved += len(pkglist)
                        for element in pkglist:
                            fd.write('%s %s/%s Removed %s (%d kiB)\n' % (date, dist.nick, repo.name, element[0], element[1]/1024))
                            msg = msg + '\n\t\t- %s (%d kiB)' % (element[0], element[1]/1024)

                    fd.close()
                    repo.changed = True

            if distnew or distremoved:
                msg = msg + '\n'
                info(1, '%s: Distribution updated (new: %d, removed: %d)' % (dist.nick, distnew, distremoved))
                sumnew = sumnew + distnew
                sumremoved = sumremoved + distremoved

    if sumnew or sumremoved:
        subject = 'changes to %s (new: %d, removed: %d)' % (os.uname()[1], sumnew, sumremoved)
        mail(subject, msg)

    if not op.generate:
        sys.exit(0)

    htmlindex()

    ### Generating metadata for available distributions/repositories
    for dist in dists:
        dist.html()

        info(1, '%s: Generating %s meta-data' % (dist.nick, dist.name))

        dist.genmetadata()
        dist.pxe()

    if cf.hardlink and not op.dists:
        hardlink(cf.srcdir)

### Unbuffered sys.stdout
sys.stdout = os.fdopen(1, 'w', 0)
sys.stderr = os.fdopen(2, 'w', 0)

### Workaround for python <= 2.2.1
try:
    True, False
except NameError:
    True, False = (0==0, 0!=0)

try:
    set()
except NameError:
    set = mySet

try:
    enumerate
except NameError:
    enumerate = lambda seq: zip(xrange(len(seq)), seq)

### Main entrance
if __name__ == '__main__':
    exitcode = 0

    op = Options(sys.argv[1:])
    cf = readconfig()
    try:
        main()
    except KeyboardInterrupt, e:
        die(6, 'Exiting on user request')
#   except OSError, e:
#       print e.errno
#       die(7, 'OSError: %s' % e)
    sys.exit(exitcode)

# vim:ts=4:sw=4:et