"""

The main program of EMirror

"""

# ----------------------------------------------------------------------
# Import functions I need 

from string import strip, find, split, lower, atoi, atof, atol, digits, rfind, capitalize, zfill, join
from sys import exit, argv, hexversion
from os import chdir, mkdir, utime, unlink, getcwd, chmod, symlink, lstat, readlink
from os import environ, rename, system, getuid, chown, remove, kill, getpid, umask
from time import time, mktime, sleep, ctime, localtime, asctime
if (hexversion != 0):  # FIXME: check this
    from ftplibbeta import FTP, all_errors, error_reply, error_temp, error_perm, error_proto
else:
    from ftplib import FTP, all_errors, error_reply, error_temp, error_perm, error_proto
from re import match, sub, compile
from types import TupleType, InstanceType, ListType
from urlparse import urlunparse
from tempfile import mktemp
from ConfigParser import ConfigParser
from error import errormsgs

from tools import *
from constants import *

import os
import getopt
import copy
import sys
import signal
import shutil
import socket
import popen2
import log4py
import process

# ----------------------------------------------------------------------

LOG_FMT = "%T %L %M"                              # LOG_FMT = log4py.FMT_DEBUG

# ----------------------------------------------------------------------
# Definition of the File-Class:

class File:
    
    def __init__(self):
        self.log4py = log4py.Logger().get_instance(self)

        self.name = ""                            # Name of the file
        self.filetype = None                      # FileType (File, Directory, Link)
        self.connection = None                    # Connection necessary for this file
        self.permissions = 0                      # File Permissions
        self.size = 0L                            # Size of the file

        self.linktarget = ""
        self.modtime = 0

    # split a file entry so that blanks are not lost, e.g. "a  b c.txt" -> ("a ", "b", "c.txt")
    def filesplit(self, string, character):
        list = []
        element = ""
        while (len(string) > 0):
            current = string[0]
            if (current == character):
                while (len(string) > 1) and (string[1] == character):
                    element = "%s%s" % (element, character)
                    string = string[1:]
                    current = string[0]
                list.append(element)
                element = ""
            else:
                element = "%s%s" % (element, current)
            string = string[1:]
        if (element != ""):
            list.append(element)
        return list

    def GetFilePerms(self, permstring):
        tempperms = 0

        # File permissions for owner
        if permstring[0] == "r":
            tempperms = tempperms | 256
        if permstring[1] == "w":
            tempperms = tempperms | 128
        if permstring[2] == "x":
            tempperms = tempperms | 64
        elif permstring[2] == "S":
            tempperms = tempperms | 2048
        elif permstring[2] == "s":
            tempperms = tempperms | 2112

        # File permissions for group
        if permstring[3] == "r":
            tempperms = tempperms | 32
        if permstring[4] == "w":
            tempperms = tempperms | 16
        if permstring[5] == "x":
            tempperms = tempperms | 8
        elif permstring[5] == "S":
            tempperms = tempperms | 1024
        elif permstring[5] == "s":
            tempperms = tempperms | 1032

        # File permissions for other
        if permstring[6] == "r":
            tempperms = tempperms | 4
        if permstring[7] == "w":
            tempperms = tempperms | 2
        if permstring[8] == "x":
            tempperms = tempperms | 1
        elif permstring[8] == "T":
            tempperms = tempperms | 512
        elif permstring[8] == "t":
            tempperms = tempperms | 513

        self.permissions = tempperms

    def SplitNonEPLF(self, filestruct):

        if (filestruct[0] == "-"):
            self.filetype = FTRegularFile
        elif (filestruct[0] == "d"):
            self.filetype = FTDirectory
        elif (filestruct[0] == "l"):
            self.filetype = FTLink
        permissionstring = filestruct[1:10]
        self.GetFilePerms(permissionstring)
        splitted = self.filesplit(filestruct, " ")

        self.name = ""
        if (lower(strip(splitted[5])) in MSMonthStrings):   # WU-FTPD < 2.6.*, ProFTPD and similar FTPD format
            self.size = atol(strip(splitted[4]))
            for i in range(8, len(splitted)):
                self.name = self.name + " " + splitted[i]
        else:                                               # WU-FTPD 2.6.* (some versions don't include the file owner)
            self.size = atol(strip(splitted[3]))
            for i in range(7, len(splitted)):
                self.name = self.name + " " + splitted[i]

        self.name = lstrip(self.name)

        if (self.filetype == FTLink):
            splitted = split(self.name, "->")
            self.name = strip(splitted[0])
            self.linktarget = strip(splitted[1])
        else:
            self.linktarget = ""

    def SplitEPLF(self, filestruct):
        splitted = split(filestruct[1:], ",")
        self.name = strip(splitted[len(splitted) - 1])
        self.linktarget = ""
        for i in range(len(splitted) - 1):
            parameter = strip(splitted[i])
            if (parameter == "/"):
                self.filetype = FTDirectory
                self.permissions = 493                               # rwxr-xr-x
            elif (parameter == "r"):
                self.filetype = FTRegularFile
                self.permissions = 420                               # rw-r--r--
            elif (parameter[0] == "s"):
                self.size = atol(parameter[1:])
            elif (parameter[0] == "m"):
                self.modtime = atoi(parameter[1:])

    def SplitMSFormat(self, filestruct):
        if (filestruct[24:29] == "<DIR>"):
            self.filetype = FTDirectory
            self.permissions = 493                                    # rwxr-xr-x
        else:
            self.filetype = FTRegularFile
            self.permissions = 420                                    # rw-r--r--
            self.size = atol(strip(filestruct[29:38]))
            
        self.name = filestruct[39:]

    def SplitFileInfo(self, filestruct, title):
        if (filestruct[0] == "+"):
            self.SplitEPLF(filestruct)
            self.log4py.debug("[ %s (debug) ] SplitFileInfo (%s): EPLF detected" % (str(title), self.name))
        elif (MSDate.match(filestruct)):
            self.SplitMSFormat(filestruct)
            self.log4py.debug("[ %s (debug) ] SplitFileInfo (%s): Microsoft (NT5 Beta ?) format detected" % (str(title), self.name))
        else:
            self.SplitNonEPLF(filestruct)
            self.log4py.debug("[ %s (debug) ] SplitFileInfo (%s): Standard unix format detected" % (str(title), self.name))

# ----------------------------------------------------------------------

class Installer:

    # Initialize the installer
    def __init__(self):
        self.index = 0                                      # The index of the source file (are ordered)
        self.title = None                                   # Mirror Title (= Installer Title)
        self.log4py = log4py.Logger().get_instance(self)
        self.sources = {}
        self.sourceregexps = {}
        self.srpms = []
        self.srpmregexp = None
        self.rpms = []
        self.rpmregexp = None
        self.rpmopts = INSRPMOpts
        self.rpmbuildopts = INSRPMBuildOpts
        self.builddir = INSBuildDir
        self.logfile = None                                 # Logfile of the build output

    # Rebuild a SRPM file
    def rebuildSRPM(self, filename):
        self.log4py.info("[ %s Installer ] Rebuilding SRPM %s" % (str(self.title), filename))
        rpms = ls(INSRPMDir, recursive = TRUE, includepath = TRUE)
        commandline = "rpm %s %s 2>&1" % (self.rpmbuildopts, filename)
        doit = popen2.popen2(commandline)
        output = doit[0].readlines()
        if (self.logfile != None):
            file = open(self.logfile, "a")
            file.write("\n[ %s Installer ] Rebuilding SRPM %s\n\n" % (str(self.title), filename))
            for i in range(len(output)):
                file.write(output[i])
            file.write("\n")
            file.close()
        newrpms = listdiff(rpms, ls(INSRPMDir, recursive = TRUE, includepath = TRUE))
        if (len(newrpms) > 0):
            self.rpms = self.rpms + newrpms
        else:
            self.log4py.error("[ %s Installer ] Rebuild of %s failed or RPM already exists. Check logfile for details." % (str(self.title), filename))

    # Install RPMs
    def installRPMs(self):
        filelist = ""
        for i in range(len(self.rpms)):
            filelist = "%s %s" % (filelist, self.rpms[i])
        self.log4py.info("[ %s Installer ] Installing RPMs: %s" % (str(self.title), strip(filelist)))

        commandline = "rpm %s %s 2>&1" % (self.rpmopts, strip(filelist))
        doit = popen2.popen2(commandline)
        output = doit[0].readlines()
        if (self.logfile != None):
            file = open(self.logfile, "a")
            file.write("\n[ %s Installer ] Installing RPMs: %s\n\n" % (str(self.title), strip(filelist)))
            for i in range(len(output)):
                file.write(output[i])
            file.write("\n")
            file.close()
        for i in range(len(output)):
            if find(output[i], "already installed") or find(output[i], "failed"):
                self.log4py.error("[ %s Installer ] Installation of RPMs failed. Check logfile for details." % str(self.title))

    # Install Source
    def installSource(self, file, commands):
        self.log4py.info("[ %s Installer ] Installing Source: %s [%s]" % (str(self.title), file, commands))

        if (find(commands, ";") != -1):
            splitted = split(commands, ";")
        else:
            splitted = [commands]

        currentdir = getcwd()
        chdir(self.builddir)

        dirlist = ls(self.builddir, includepath = TRUE, onlydirectories = TRUE)
        newdir = []
        filelog = open(self.logfile, "a")
        for i in range(len(splitted)):
            command = splitted[i]
            command = sub("%f", file, command)
            if (len(newdir) == 1):
                command = sub("%d", newdir[0], command)
            self.log4py.debug("[ %s Installer (debug) ] Executing: %s" % (str(self.title), command))
            filelog.write("\n[ %s Installer ] Source: Executing %s\n\n" % (str(self.title), command))
            if (command[:2] == "cd"):
                chdir(command[3:])
            else:
                doit = popen2.popen2("%s 2>&1" % command)
                output = doit[0].readlines()
                for i in range(len(output)):
                    filelog.write(output[i])
                filelog.write("\n")
            if (len(newdir) == 0):
                newdir = listdiff(dirlist, ls(self.builddir, includepath = TRUE, onlydirectories = TRUE))

        filelog.close()
        if (len(newdir) == 1):                              # Cleanup (remove build directory)
            rm(newdir[0])
        chdir(currentdir)

    # Do the real installation
    def finish(self):
        if (self.logfile != None) and (os.path.exists(self.logfile)): unlink(self.logfile)
        for i in range(len(self.srpms)):
            self.rebuildSRPM(self.srpms[i])
        if (len(self.rpms) > 0):
            self.installRPMs()
        keys = self.sources.keys()
        keys.sort()
        for i in range(len(keys)):
            key = keys[i]
            self.installSource(self.sources[key][0], self.sources[key][1])

    # Check wether a file has to be installed or not
    def checkfile(self, path, filename, localname):
        if (self.rpmregexp != None) and ((self.rpmregexp.match(filename)) or (self.rpmregexp.match(path + filename))):
            self.rpms.append(localname)
        if (self.srpmregexp != None) and ((self.srpmregexp.match(filename)) or (self.srpmregexp.match(path + filename))):
            self.srpms.append(localname)
        keys = self.sourceregexps.keys()
        keys.sort()
        for i in range(len(keys)):
            key = keys[i]
            if (self.sourceregexps[key][0].match(filename)) or (self.sourceregexps[key][0].match(path + filename)):
                self.sources[self.index] = [localname, self.sourceregexps[key][1]]
                self.index = self.index + 1

    # Add regular expressions to the installer (from the config file)
    def add(self, option, value):
        if (lower(option) == "rpmoptions"):
            self.rpmopts = value
        elif (lower(option) == "rpmbuildoptions"):
            self.rpmbuildopts = value
        elif (lower(option) == "logfile"):
            self.logfile = value
        elif (lower(option)[:4] == "rpms"):
            self.rpmregexp = compile(unfold(value))
        elif (lower(option)[:5] == "srpms"):
            self.srpmregexp = compile(unfold(value))
        elif (lower(option)[:5] == "other"):
            index = atoi(option[5:])
            filename = value[:find(value, "[") - 1]
            command = value[find(value, "[") + 1:-1]
            self.sourceregexps[index] = [compile(unfold(filename)), command]
        elif (lower(option) == "builddir"):
            self.builddir = value

# ----------------------------------------------------------------------

class Connection:
    def __init__(self):
        self.protocoll = None        # Protocoll to connect
        self.hostname = None         # Hostname of the remote host
        self.realhostname = ""       # real Hostname / IP (for hidden sites)
        self.port = None             # Port to connect to
        self.username = None         # Username
        self.password = None         # Password used ...
        self.status = None           # Connection status

# ----------------------------------------------------------------------
# Definition of the Mirror-Class:

class Mirror:

    # Initialisation of some variables
    def __init__(self):
        self.url = ""                             # Full URL of the mirror
        self.protocol = ""                        # Protocol to start with
        self.username = ""                        # Username
        self.password = ""                        # Password
        self.hostname = ""                        # Hostname / IP of the remote site
        self.realhostname = ""                    # real Hostname / IP (for hidden sites)
        self.path = ""                            # Remote directory to mirror
        self.directory = getcwd() + "/"           # Local directory
        self.noexception = FALSE                  # Do NOT use exceptions ?
        self.log4py = log4py.Logger().get_instance(self)
        self.test = FALSE                         # Test mode
        self.port = 0                             # Port Number
        self.delay = DVDelay                      # Delay time between retries

        self.followsymlinks = FALSE               # Do not follow symbolic links
        self.parentformat = PFFull                # Full parent directory format

        self.logdirectory = ""                    # Directory where the logfiles reside
        self.logfilename = None                   # Name of the logfile
        self.logfileformat = LFPlain              # Format of the logfiles (Plain, HTML)
        self.logdata = []                         # The Stuff which should appear in the log

        self.retries = DVRetries                  # Number of retries
        self.logfile = None                       # Filehandler of the logfile
        self.recursive = FALSE                    # Recursive download
        self.starttime = 0                        # Start time of the mirroring
        self.endtime = 0                          # End time of the mirroring
        self.category = "Assorted"                # Category of the mirror (for logfiles)
        self.description = None                   # Description of the mirror
        self.adminemail = None                    # EMail-adress of the ftp-maintainer
        self.errorcolor = "#CC0033"               # Default color for error messages
        self.warningcolor = "#191970"             # Default color for warning messages

        self.logtemplate = None                   # Template file for logs
        self.template = None                      # Template data
        self.templateloop = None                  # Template data for 

        self.maxdays = 0                          # maximum age of files to download (days)
        self.maxage = 0                           # maximum age of local files (days)
        self.maxdelete = None                     # maximum size or percantage of files to delete

        self.onlylatest = []                      # Only get the latest version
        self.title = None                         # Title of the Mirror
        self.localurl = None                      # Local download URL
        self.exclude = None                       # Exclude pattern
        self.include = None                       # Regular expression for file selection
        self.ignore = None                        # Regular expression to ignore local files
        self.deletelocal = FALSE                  # Always delete local files, which don't exist on the remote side
        self.errormailcmd = None                  # Command to execute on error
        self.infomailcmd = None                   # Command to execute when a mirror has changed
        self.ignoremdtm = FALSE                   # Do not ignore the MDTM when downloading files
        self.passivemode = FALSE                  # Use passive mode ?
        self.continueftp = FALSE                  # Continue FTP downloads ?
        self.umask = None

        self.notloggedin = FALSE                  # default: emirror could login

        self.filelist = []                        # List of files for iterative checking
        self.connection = None                    # Current connection
        self.ftp = FTP()                          # The FTP-Connection

        self.downloadfile = ""                    # Name of the file which is being downloaded
        self.currentfile = ""                     # the current file being processed

        self.remotefiles = []                     # Remote file listing for a new directory
        self.localfiles = []                      # Local file list for a new directory

        self.statfilelist = []                    # File-list for the MyStat-Function

        self.nrbytesretr = 0L                     # Number of bytes retrieved
        self.nrfilesretr = 0                      # Number of files retrieved
        self.nrdirscreated = 0                    # Number of directories created
        self.nrlinkscreated = 0                   # Number of links created
        self.nrbytesdel = 0L                      # Number of bytes deleted
        self.nrfilesdel = 0                       # Number of files deleted
        self.nrdirsdel = 0                        # Number of directories deleted
        self.nrlinksdel = 0                       # Number of links deleted

        self.owner = None                         # Set owner of downloaded files to this UID
        self.group = None                         # Set group of downloaded files to this GID

        self.filelistcache = {}                   # Simple File Listing cache
        self.installer = Installer()              # File Installer

        self.proxy = None        		  # Proxy host:port

    # Open the logfile ... that is notice the start time ;-)
    def StartLog(self):
        self.starttime = time()

    # Append a line to the logfile
    def AppendLog(self, line):
        if self.logfilename != None:

            if (line[0] == LMRetrieve) and (self.localurl != None):
                downloadurl = self.GetLCwd(parentdirectory = self.localurl, hostname = self.hostname) + self.currentfile.name
                downloadname = self.currentfile.path + self.currentfile.name
                if (self.logfileformat == LFPlain):
                    logrow = [ line[0], downloadname ]
                else:
                    logrow = [ line[0], "<A HREF=\"%s\">%s</A>" % (downloadurl, downloadname) ]
            else:
                logrow = [ line[0], line[1] ]

            self.logdata.append(logrow)

    # Read the logtemplate file
    def ReadLogTemplate(self):
        self.log4py.debug("[ %s (debug) ] Logfile: reading template file" % str(self.title))
        file = open(self.logtemplate, "r")
        line = "<empty>"
        self.template = []
        self.templateloop = []
        target = self.template

        if (self.logfileformat == LFHTML):
            dateiname = "index" + LEHTML
            fileupdatedname = "updated" + LEHTML
            fileerrorsname = "errors" + LEHTML
        elif (self.logfileformat == LFPHP):
            dateiname = "index" + LEPHP
            fileupdatedname = "updated" + LEPHP
            fileerrorsname = "errors" + LEPHP
        elif (self.logfileformat == LFPlain):
            dateiname = "index" + LEPlain
            fileupdatedname = "updated" + LEPlain
            fileerrorsname = "errors" + LEPlain

        while (line != ""):
            line = file.readline()
            line = sub("\$indexfilename\$", dateiname, line)
            line = sub("\$updatedfilename\$", fileupdatedname, line)
            line = sub("\$errorfilename\$", fileerrorsname, line)
            if (lower(strip(line)) == "$logstart$"):
                target.append("$logdata$")
                target = self.templateloop
            elif (lower(strip(line)) == "$logend$"):
                target = self.template
            elif (find(strip(lower(line)), "<!-- errorcolor:") != -1):
                color = strip(lower(line))
                position = find(color, "<!-- errorcolor:")
                color = color[position + len("<!-- errorcolor:"):]
                position = find(color, "-->")
                color = strip(color[:position])
                self.errorcolor = color
            elif (find(strip(lower(line)), "<!-- warningcolor:") != -1):
                color = strip(lower(line))
                position = find(color, "<!-- warningcolor:")
                color = color[position + len("<!-- warningcolor:"):]
                position = find(color, "-->")
                color = strip(color[:position])
                self.warningcolor = color
            else:
                target.append(line)
        file.close()

    # Check the logfilename and adopt its extension
    def CheckLogfilename(self, filename):
        if (rfind(filename, ".") != -1):
            extension = lower(filename[rfind(filename, "."):])
            basicname = filename[:rfind(filename, ".")]
        else:
            extension = ""
            basicname = filename
        if (extension == "") or (extension in [".html", ".asc", ".txt", ".php", ".php3", ".php4", ".htm"]):
            if (self.logfileformat == LFHTML):
                return basicname + LEHTML
            elif (self.logfileformat == LFPHP):
                return basicname + LEPHP
            elif (self.logfileformat == LFPlain):
                return basicname + LEPlain
        else:
            return filename
        
    # Write the whole log file at once
    def WriteLog(self):
        if self.logfilename != None:
            self.log4py.info("[ %s ] Logfile: writing" % str(self.title))

            if (not os.path.exists(self.logdirectory)):
                try:
                    mkdirtree(self.logdirectory)
                    self.log4py.info("[ %s ] Logfile: Directory \"%s\" created" % (str(self.title), self.logdirectory))
                except:
                    self.log4py.error("[ %s ] Logfile: Error: Couldn't create directory \"%s\" !" % (str(self.title), self.logdirectory))
                    return
            try:
                newFilename = self.CheckLogfilename(self.logfilename)
                self.logfile = open(self.logdirectory + newFilename, "w")
            except:
                self.log4py.error("[ %s ] Logfile: Error: Couldn't write logfile \"%s\" !" % (str(self.title), self.logfilename))
                return

            self.endtime = time()
            difference = long(self.endtime) - long(self.starttime)
            
            if (self.logtemplate == None):
                self.log4py.error("[ %s ] Logfile: Error: No template file specified !" % str(self.title))
                return

            # Read the template file
            self.ReadLogTemplate()

            # Unparse the URL for a nice outfit :)
            url = urlunparse((self.protocol, self.hostname, self.path, "", "", ""))

            # Check wether a local download-URL is set
            if (self.localurl != None):
                downloadurl = self.GetLCwd(hostname = self.hostname, path = self.path, parentdirectory = self.localurl)
            else:
                downloadurl = ""

            # Check the username and password
            if (lower(self.username) == "ftp") or (lower(self.username) == "anonymous"):
                username = self.username
                password = self.password
            else:
                username = "-- Hidden --"
                password = "-- Hidden --"

            # Check the administrator EMail address
            if self.adminemail != None:
                email = self.adminemail
            else:
                email = ""

            # Calculate the time
            totalseconds = difference
            minutes, seconds = divmod(totalseconds, 60)
            hours, minutes = divmod(minutes, 60)

            # Create Format specific text
            if (self.logfileformat == LFPlain):
                duration = "%2d Hour(s), %2d Minute(s) and %2d Second(s)" % (hours, minutes, seconds)
                downloaded = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
                deleted = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)
            else:
                duration = "<b>%2d</b> Hour(s), <b>%2d</b> Minute(s) and <b>%2d</b> Second(s)" % (hours, minutes, seconds)
                downloaded = "<b>%s</b> Byte(s) in <b>%.0f</b> File(s), <b>%.0f</b> Link(s) and <b>%.0f</b> Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
                deleted = "<b>%s</b> Byte(s) in <b>%.0f</b> File(s), <b>%.0f</b> Link(s) and <b>%.0f</b> Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)

            # Calculate download rate
            if (self.nrbytesretr > 0):
                if (totalseconds > 0):
                    rate = self.nrbytesretr / totalseconds
                else:
                    rate = 0
                kbytepersec = int(rate / 1024)
                if (kbytepersec > 0):
                    if (self.logfileformat == LFPlain):
                        downloadrate = "%.0f KByte/sec" % int(kbytepersec)
                    else:
                        downloadrate = "<b>%.0f</b> KByte/sec" % int(kbytepersec)
                else:
                    if (self.logfileformat == LFPlain):
                        downloadrate = "%.0f Byte/sec" % int(rate)
                    else:
                        downloadrate = "<b>%.0f</b> Byte/sec" % int(rate)
            else:
                downloadrate = "Nothing downloaded"

            # Calculate the entry for STATUS
            if (self.nrfilesretr == 0) and (self.nrfilesdel == 0) and (self.nrlinksdel == 0) and (self.nrdirscreated == 0) and (self.nrdirsdel == 0) and (self.nrlinkscreated == 0):
                if (self.notloggedin == FALSE):
                    status = MSunchanged
                    self.logdata.append([ "Information", "There's nothing new on the remote host" ])
                else:
                    status = MSnotloggedin
            else:
                status = str(self.endtime)

            statsstring = "%.0f %.0f %.0f %.0f %.0f %.0f %.0f %.0f %.0f %.0f" % (self.nrbytesretr, self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated, self.nrbytesdel, self.nrfilesdel, self.nrlinksdel, self.nrdirsdel, self.starttime, self.endtime)

            # Process the template
            for i in range(len(self.template)):
                line = self.template[i]
                if (line == "$logdata$"):
                    for j in range(len(self.logdata)):
                        loginfo = self.logdata[j]
                        for k in range(len(self.templateloop)):
                            line = self.templateloop[k]
                            if (self.logfileformat != LFPlain) and (loginfo[0] == LMError):
                                line = sub("\$type\$", "<font color=\"%s\">%s</font>" % (self.errorcolor, loginfo[0]), line)
                                line = sub("\$message\$", "<font color=\"%s\">%s</font>" % (self.errorcolor, loginfo[1]), line)
                            elif (self.logfileformat != LFPlain) and (loginfo[0] == LMWarning):
                                line = sub("\$type\$", "<font color=\"%s\">%s</font>" % (self.warningcolor, loginfo[0]), line)
                                line = sub("\$message\$", "<font color=\"%s\">%s</font>" % (self.warningcolor, loginfo[1]), line)
                            else:
                                line = sub("\$type\$", loginfo[0], line)
                                line = sub("\$message\$", loginfo[1], line)
                            self.logfile.write(line)
                else:
                    line = sub("\$title\$", str(self.title), line)
                    line = sub("\$url\$", url, line)
                    line = sub("\$category\$", str(self.category), line)
                    line = sub("\$description\$", str(self.description), line)
                    line = sub("\$localurl\$", downloadurl, line)
                    line = sub("\$email\$", email, line)
                    line = sub("\$hostname\$", self.hostname, line)
                    line = sub("\$port\$", str(self.port), line)
                    line = sub("\$username\$", username, line)
                    line = sub("\$password\$", password, line)
                    line = sub("\$remotedirectory\$", self.path, line)
                    line = sub("\$localdirectory\$", self.directory, line)
                    line = sub("\$tries\$", str(self.retries), line)
                    line = sub("\$delay\$", str(self.delay), line)
                    line = sub("\$downloadrate\$", downloadrate, line)
                    line = sub("\$starttime\$", ctime(self.starttime), line)
                    line = sub("\$endtime\$", ctime(self.endtime), line)
                    line = sub("\$duration\$", duration, line)
                    line = sub("\$downloadstats\$", downloaded, line)
                    line = sub("\$deletestats\$", deleted, line)
                    line = sub("\$mdtm\$", status, line)
                    line = sub("\$statistics\$", statsstring, line)
                    line = sub("\$lastupdate\$", ctime(time()), line)
                    line = sub("\$timestamp\$", str(time()), line)
                    self.logfile.write(line)

            if (status != MSunchanged) and (status != MSnotloggedin) and (self.infomailcmd != None):
                self.log4py.info("[ %s ] Executing: %s" % (str(self.title), self.infomailcmd))
                if find(self.infomailcmd, "%s") != -1:
                    cmd = self.infomailcmd % str(self.title)
                else:
                    cmd = self.infomailcmd
                filename = mktemp()
                file = open(filename, "w")
                file.write("\nECLiPt-Mirror: Package \"%s\" updated\n\n" % str(self.title))
                file.write("[        Type ][ Message                                                      ]\n")
                for i in range(len(self.logdata)):
                    file.write("%s    %s\n" % (pfill(strip(self.logdata[i][0]), " ", 13), self.logdata[i][1]))
                file.write("\n")
                file.write("Generated at %s" % asctime(localtime(time())))
                file.close()
                system("cat %s | %s" % (filename, cmd))
                unlink(filename)
                
            self.logfile.close()
                
    # Check wether a local directory exists & create, if not
    def CheckDirectory(self, directory):

        if (directory[-1:] == "/"):
            directory = directory[:-1]
        if (os.path.exists(directory)) or (os.path.islink(directory)):
            if (os.path.islink(directory)) or (os.path.isfile(directory)):
                unlink(directory)
        if not os.path.exists(directory):
            pwd = getcwd()
            splitted = split(directory, "/")
            path = "/"
            for i in range(len(splitted)):
                if (os.path.exists(path + splitted[i])) and (os.path.isfile(path + splitted[i])):
                    unlink(path + splitted[i])
                if (not os.path.exists(path + splitted[i])) and (not os.path.islink(path + splitted[i])):
                    try:
                        chdir(path)
                    except:
                        self.log4py.error("[ %s ] Error: couldn't change to directory %s" % (str(self.title), directory))
                        exit(1)
                    try:
                        mkdir(splitted[i])
                        if (self.owner != None):
                            chown(splitted[i], self.owner, self.group)
                    except:
                        self.log4py.error("[ %s ] Error: Couldn't create directory %s" % (str(self.title), directory))
                        exit(1)
                    self.nrdirscreated = self.nrdirscreated + 1
                path = path + splitted[i] + "/"
            chdir(pwd)

    # Create the correct linktargetname and path from (name, path, target)
    def AdoptLinkTarget(self, name, path, target):

        if (target[0] != "/"):                              # it's a relative link -> adopt the linktarget
            newpath = path
            newname = target
            if newpath[-1:] != "/":
                newpath = newpath + "/"
            if (newname[0:3] == "../"):
                newpath = newpath[:-1]
                while (newname[0:3] == "../"):
                    newname = newname[3:]
                    newpath = newpath[0:rfind(newpath, "/")]
                newpath = newpath + "/"
                if (rfind(newname, "/") != -1):
                    newpath = newpath + newname[:rfind(newname, "/") + 1]
                    newname = newname[rfind(newname, "/") + 1:]
            while (find(newname, "/") != -1):
                newpath = newpath + newname[:find(newname, "/") + 1]
                newname = newname[find(newname, "/") + 1:]
        else:                                               # the linktarget is a absolut link
            newpath = target
            newname = ""

        return (newname, newpath)
    
    # Create a symbolic link
    def CreateLink(self):

        self.currentfile.linktarget = sub("//", "/", self.currentfile.linktarget)

        createlink = TRUE                                        # Default: create link
        append = TRUE                                            # Append file to the list (only if followsymlink is true)

        newFile = File()
        newFile.log4py.set_loglevel(self.log4py.get_loglevel())
        newFile.filetype = FTLinkTarget

        # Check the Linktarget and modify name & path if necessary

        newFile.name, newFile.path = self.AdoptLinkTarget(self.currentfile.name, self.currentfile.path, self.currentfile.linktarget)
        
        newFile.connection = self.currentfile.connection

        # Check Include / Exclude rules
        
        if (self.parentformat == PFNone) and ((len(newFile.path) < len(self.path)) or (newFile.path[:len(self.path)] != self.path)):
            self.log4py.warn("[ %s ] Warning: Link %s%s is outside the parent directory !" % (str(self.title), newFile.path, newFile.name))
            createlink = FALSE
        else:

            # Include rules
            if (self.include != None) and ((self.include.match(newFile.name) == None) and (self.include.match(newFile.path + newFile.name) == None)):
                createlink = FALSE

            # Exclude rules
            if ((self.exclude != None) and ((self.exclude.match(newFile.name) != None) or self.exclude.match(newFile.path + newFile.name) != None)):
                createlink = FALSE

            # Ignore rules
            if ((self.ignore != None) and ((self.ignore.match(newFile.name) != None) or self.ignore.match(newFile.path + newFile.name) != None)):
                createlink = FALSE

            if ((self.currentfile.path + self.currentfile.name) == (newFile.path + newFile.name)): # Check wether the link points to itself (endless loop)
                self.log4py.warn("[ %s ] Warning: Link %s%s points to itself !" % (str(self.title), self.currentfile.path, self.currentfile.name))
                append = FALSE

            if (newFile.name == "."):                                 # Check wether the link points to the current directory (endless loop)
                self.log4py.warn("[ %s ] Warning: Link %s%s points to . (current directory) !" % (str(self.title), self.currentfile.path, self.currentfile.name))
                createlink = FALSE

        if (self.followsymlinks == TRUE) and (createlink == TRUE) and (append == TRUE): # Don't append the file if you don't want to follow links
            self.filelist.append(newFile)

        pwd = getcwd()
        directory = self.GetLCwd()
        chdir(directory)

        deletelink = FALSE
            
        if os.path.islink(self.currentfile.name):                     # check wether the link already exists
            linktarget = readlink(self.currentfile.name)
            if (linktarget != self.currentfile.linktarget):           # if the linktarget has changed -> delete the link
                deletelink = TRUE
            if (createlink == FALSE):                                 # if the link exists, but shouldn't -> delete it
                deletelink = TRUE                                     # This makes the mirror clean (hopefully without dead links)

        elif os.path.exists(self.currentfile.name):       	      # this is, if a directory or file has become a link ... delete the old stuff then
            deletelink = TRUE

        if (os.path.islink(self.currentfile.name) and (createlink == FALSE)) or (deletelink == TRUE): # The link exists, but it shouldn't
            self.log4py.info("[ %s ] Deleting Link / File / Directory (FTP): %s%s " % (str(self.title), self.currentfile.path, self.currentfile.name))
            result = rm(self.currentfile.name)
            self.AppendLog([LMDelete, self.currentfile.path + self.currentfile.name])
            self.nrdirsdel = self.nrdirsdel + result[0]
            self.nrfilesdel = self.nrfilesdel + result[1]
            self.nrbytesdel = self.nrbytesdel + result[2]
            self.nrlinksdel = self.nrlinksdel + result[3]
            
        if (createlink == TRUE) and (not os.path.islink(self.currentfile.name)):
            self.log4py.info("[ %s ] Creating Link (FTP): %s%s -> %s" % (str(self.title), self.currentfile.path, self.currentfile.name, self.currentfile.linktarget))
            
            try:
                symlink(self.currentfile.linktarget, self.currentfile.name)
            except os.error, detail:
                self.log4py.warn("[ %s ] Warning: couldn't create link %s%s -> %s: %s" % (
                    str(self.title), self.currentfile.path, self.currentfile.name, self.currentfile.linktarget, detail))
            self.AppendLog([LMLink, self.currentfile.path + self.currentfile.name + " -> " + self.currentfile.linktarget])
            self.nrlinkscreated = self.nrlinkscreated + 1

        chdir(pwd)

    # Get Remote Files
    def GetRemoteFiles(self):
        self.log4py.debug("[ %s (debug) ] Getting list of remote files" % str(self.title))
        files2download = self.remotefiles
        for i in range(len(files2download)):
            remotefile = files2download[i]
            download = FALSE
            if (remotefile.filetype == FTDirectory):
                if (self.recursive == TRUE):
                    download = TRUE
            elif (remotefile.filetype == FTRegularFile) or (remotefile.filetype == FTLink):
                download = TRUE

            if (download == TRUE):
                # Insert file at the beginning to reduze memory usage
                if (remotefile.filetype == FTLink) or (remotefile.filetype == FTRegularFile):
                    self.filelist.insert(0, remotefile)
                else:
                    self.filelist.append(remotefile)

    # Delete a file or subtree
    def DeleteFile(self, path, directory, filename, reason = ""):

        # Do not delete which are being ignored !
        if (self.ignore != None) and ((self.ignore.match(filename) != None) or (self.ignore.match(directory + filename) != None)):
            return
        else:

            if (self.maxdelete > -1):
                if (self.nrbytesdel < self.maxdelete):
                    if (self.test == FALSE):
                        self.log4py.info("[ %s ] Deleting: %s%s (max %s Byte) %s" % (str(self.title), path, filename, lng2str(self.maxdelete), reason))
                    else:
                        self.log4py.info("[ %s (Test) ] Deleting: %s%s (max %s Byte) %s" % (str(self.title), path, filename, lng2str(self.maxdelete), reason))                            
            else:
                if (self.test == FALSE):
                    self.log4py.info("[ %s ] Deleting: %s%s %s" % (str(self.title), path, filename, reason))
                else:
                    self.log4py.info("[ %s (Test) ] Deleting: %s%s %s" % (str(self.title), path, filename, reason))
            try:
                if (self.nrbytesdel < self.maxdelete) or (self.maxdelete <= -1):
                    if (self.test == FALSE):
                        result = rm(directory + filename, self.maxdelete, self.nrbytesdel)
                    else:
                        result = [0, 0, 0, 0]
                else:
                    result = [0, 0, 0, 0]
            except OSError, detail:
                self.HandleError(detail, ESDeleteFile, extra = "filename / directory: \"%s/%s\")" % (directory, filename))
                return FALSE
            
            self.AppendLog([LMDelete + " " + reason,  path + filename])
            self.nrdirsdel = self.nrdirsdel + result[0]
            self.nrfilesdel = self.nrfilesdel + result[1]
            self.nrbytesdel = self.nrbytesdel + result[2]
            self.nrlinksdel = self.nrlinksdel + result[3]

    # Delete local files, which don't exist on the remote site any more 
    def DelOldLocalFiles(self):
        self.log4py.debug("[ %s (debug) ] Deleting old local files" % str(self.title))
        directory = self.GetLCwd()
        if (self.maxage != 0):
            maxmdtm = long(time() - (self.maxage * 60 * 60 * 24))

        localfilenames = []                                          # Remove files which are older than maxage days
        for i in range(len(self.localfiles)):
            if (self.maxage != 0):
                filename = directory + "/" + self.localfiles[i]
                if (os.path.isfile(filename) and (not os.path.islink(filename))):
                    if (lstat(filename)[8] < maxmdtm):
                        self.DeleteFile(self.currentfile.path, directory, self.localfiles[i], "(file older than " + str(self.maxage) + " day(s))")
                    else:
                        localfilenames.append(self.localfiles[i])
                else:
                    localfilenames.append(self.localfiles[i])
            else:
                localfilenames.append(self.localfiles[i])

        remotefilenames = []
        for i in range(len(self.remotefiles)):
            if (self.remotefiles[i].filetype == FTDirectory):
                remotefilenames.append(self.remotefiles[i].path[rfind(self.remotefiles[i].path, "/", 0, -2) + 1:][:-1])
            else:
                remotefilenames.append(self.remotefiles[i].name)
        remotefilenames.sort()

        for i in range(len(localfilenames)):
            filename = localfilenames[i]
            
            if (not filename in remotefilenames):
                if (not (self.IsTempFilename(filename) and (self.RealFilename(filename) in remotefilenames))) or (self.continueftp == FALSE):
                    if (self.include == None) or (self.deletelocal == TRUE) or ((self.include != None) and ((self.include.match(filename) != None) or (self.include.match(self.currentfile.path + filename) != None))):
                        self.DeleteFile(self.currentfile.path, directory, filename)

    # Get the local filelist
    def GetLocalFileList(self):
        self.log4py.debug("[ %s (debug) ] Getting list of local files" % (str(self.title)))
        directory = self.GetLCwd()
        try:
            self.localfiles = listdir(directory)
        except OSError, detail:
            self.localfiles = []
            self.HandleError(detail, ESGetLocalFileList, extra = "listdir(\"%s\")" % directory)

    # Callback for listing files
    def LsCallBack(self, line):
        if (line != "") and (lower(line) != "ls: .: permission denied") and (line[0] in [ "-", "d", "l", "+" ]) or (MSDate.match(line)):
            newfile = File()
            newfile.log4py.set_loglevel(self.log4py.get_loglevel())
            newfile.SplitFileInfo(line, self.title)
            if (newfile.name != ".") and (newfile.name != ".."):
                self.remotefiles.append(newfile)

    # Get the remote filelist
    def GetRemoteFileList(self, directory = "", cache = TRUE, exclude = TRUE):
        if (directory == ""):
            directory = self.currentfile.path
        if self.filelistcache.has_key(directory) and (cache == TRUE):
            self.log4py.info("[ %s ] Get Filelist: %s (cached)" %(str(self.title), directory))
            self.remotefiles = copy.copy(self.filelistcache[directory])
        else:
            self.log4py.info("[ %s ] Get Filelist: %s" % (str(self.title), directory))

            if (exclude == FALSE):
                tempinclude = self.include
                tempexclude = self.exclude
                self.include = self.exclude = None

            self.remotefiles = []

            # Change to the directory
            if (self.noexception == TRUE):
                self.ftp.cwd(directory)
            else:
                try:
                    self.ftp.cwd(directory)
                except all_errors, detail:
                    self.HandleError(detail, ESGetRemoteFileListCwd, extra = "ftp.cwd(\"%s\")" % directory)
                    return FALSE

            # Get the filelist
            if (self.noexception == TRUE):
                response = self.ftp.retrlines("LIST -a", self.LsCallBack)
            else:
                try:
                    # self.ftp.dir(self.LsCallBack)                  # This doesn't receive hidden directories
                    response = self.ftp.retrlines("LIST -a", self.LsCallBack)
                except (error_reply, error_temp, error_perm, error_proto, socket.error, IOError), detail:
                    self.HandleError(detail, ESGetRemoteFileListLS, extra = "ftp.dir(\"%s\")" % directory)
                    return FALSE
                except EOFError, detail:
                    self.HandleError("901 ftplib internal: connection lost", ESGetRemoteFileListLS, extra = "ftp.dir(\"%s\")" % directory)
                    return FALSE

            for i in range(len(self.remotefiles)):                   # Fixup name and path of all files
                if (self.remotefiles[i].filetype == FTDirectory):
                    self.remotefiles[i].path = directory + self.remotefiles[i].name + "/"
                    self.remotefiles[i].name = ""
                else:
                    self.remotefiles[i].path = directory
                self.remotefiles[i].connection = self.currentfile.connection

            newremotefiles = []                                      # Remove files, which don't match include or exclude
            for i in range(len(self.remotefiles)):

                remove = FALSE

                if (self.remotefiles[i].filetype == FTDirectory):
                    fullname = self.remotefiles[i].path[:-1]
                    filename = fullname[rfind(fullname, "/") + 1:]
                else:
                    filename = self.remotefiles[i].name
                    fullname = directory + filename

                # Rules for including files
                if (self.include != None) and ((self.include.match(filename) == None) and (self.include.match(fullname) == None)):
                    remove = TRUE

                # Rules for excluding files
                if (self.exclude != None) and ((self.exclude.match(filename) != None) or (self.exclude.match(fullname) != None)):
                    remove = TRUE

                # Rules for ignoreing files
                if (self.ignore != None) and ((self.ignore.match(filename) != None) or (self.ignore.match(fullname) != None)):
                    remove = TRUE
                
                if (remove == FALSE):
                    newremotefiles.append(self.remotefiles[i])

            self.remotefiles = newremotefiles
            
            if (exclude == FALSE):
                self.include = tempinclude
                self.exclude = tempexclude

            if (cache == TRUE):
                self.filelistcache[directory] = copy.copy(self.remotefiles)

    # Do some error handling
    def HandleError(self, error, source, extra = None):
        if (str(error) != ""):
            if (type(error) == TupleType) or (type(error) == InstanceType):
                errornumber = atoi(str(error[0]))
                errormessage = str(error[1])
            else:
                errornumber = atoi(str(error)[0:3])
                errormessage = strip(str(error)[4:])
        else:
            errornumber = 999
            errormessage = "emirror internal: unknown error"
        self.log4py.error("[ %s ] Error: #%d - %s" % (str(self.title), errornumber, errormessage)) 
        found = FALSE
        for i in range(len(errormsgs)):
            oneerror = errormsgs[i]
            if (errornumber == oneerror[0]) and (find(lower(errormessage), lower(oneerror[1])) != -1):
                found = TRUE
                if (oneerror[2] == EAContinue):
                    self.AppendLog([LMError, self.currentfile.path + self.currentfile.name + ": " + str(errornumber) + "-" + errormessage])
                elif (oneerror[2] == EAAbort):
                    self.End(1)
                elif (oneerror[2] == EARetry):
                    self.ReconnectFTP()

        if (found == FALSE):
            self.log4py.error("%s" % PIHeader)
            self.log4py.error("          Path: %s" % self.currentfile.path)
            self.log4py.error("          Name: %s" % self.currentfile.name)
            self.log4py.error("        Source: %s" % str(source))
            self.log4py.error("         Error: %s" % str(error))
            self.log4py.error("   Type(Error): %s" % str(type(error)))
            if (type(error) == InstanceType):
                self.log4py.error("    Dictionary: %s" % str(error.__dict__))
            self.log4py.error("   Errornumber: %d" % errornumber)
            self.log4py.error("  Errormessage: %s" % errormessage)
            self.log4py.error("         Extra: %s\n" % str(extra))

            if (self.errormailcmd != None):
                self.log4py.info("Executing: %s" % self.errormailcmd)
                filename = mktemp()
                file = open(filename, "w")
                file.write("%s" % PIHeader)
                file.write("\nError Report (%s):\n\n" % str(self.title))
                file.write("   Admin EMail: %s\n" % str(self.adminemail))
                file.write("    Mirror URL: %s\n" % str(self.url))
                file.write("          Path: %s\n" % self.currentfile.path)
                file.write("          Name: %s\n" % self.currentfile.name)
                file.write("        Source: %s\n" % str(source))
                file.write("         Error: %s\n" % str(error))
                file.write("   Type(Error): %s\n" % str(type(error)))
                if (type(error) == InstanceType):
                    file.write("    Dictionary: %s\n" % str(error.__dict__))
                file.write("   Errornumber: %d\n" % errornumber)
                file.write("  Errormessage: %s\n" % errormessage)
                file.write("         Extra: %s\n\n" % str(extra))
                file.write("Please try and correct this error as soon as possible.\n\n")
                file.close()
                system("cat %s | %s" % (filename, self.errormailcmd))
                unlink(filename)
            else:
                self.log4py.error("Please report this error to Martin.Preishuber@eclipt.at")
                self.log4py.error("or check error.py and adapt it (it's not really difficult ;-))")

            self.AppendLog([LMError, str(errornumber) + "-" + errormessage])
            self.WriteLog()
            exit(1)

    # Connect to a Website
    def ConnectHTTP(self):
        self.log4py.warn("[ %s ] Connecting (HTTP): not supported yet (use wget meanwhile)" % str(self.title))

    # Disconnect from a Website
    def DisconnectHTTP(self):
        self.log4py.warn("[ %s ] Disconnecting (HTTP): not supported yet (use wget meanwhile)" % str(self.title))

    # Connect to a FTP-site
    def ConnectFTP(self):
        if self.connection.realhostname != "":
            hostname = self.connection.realhostname
        else:
            hostname = self.connection.hostname
        self.log4py.info("[ %s ] Connecting (FTP): %s port %d" % (str(self.title), hostname, self.connection.port))
        nrretries = 1
        errormessage = None
        successful = FALSE
        if (self.retries == 0):
            unlimited = TRUE
        else:
            unlimited = FALSE
        while (((nrretries <= self.retries) or (unlimited == TRUE)) and (successful == FALSE)):
            if (self.proxy == None):
                if (nrretries == 1):
                    self.log4py.info("[ %s ] FTP login: first try (out of %d)" % (str(self.title), self.retries))
                else:
                    self.log4py.info("[ %s ] FTP login: retry #%d (out of %d) [%s]" % (str(self.title), nrretries - 1, self.retries, str(errormessage)))
            elif (self.proxy != None):
                if (nrretries == 1):
                    self.log4py.info("[ %s ] FTP login (proxy: %s): first try (out of %d)" % (str(self.title), self.proxy, self.retries))
                else:
                    self.log4py.info("[ %s ] FTP login (proxy: %s): retry #%d (out of %d) [%s]" % (str(self.title), self.proxy, nrretries - 1, self.retries, str(errormessage)))
 
            try:

                # Connect via a Proxy host
                if (self.proxy != None):
                    proxyhost, proxyport = split(self.proxy, ":")
                    self.ftp.connect(proxyhost, atoi(proxyport))
                    self.ftp.login("%s@%s:%d" % (self.connection.username, hostname, self.connection.port), self.connection.password)
                    successful = TRUE

                # Connect directly
                else:
                    self.ftp.connect(hostname, self.connection.port)
                    self.ftp.login(self.connection.username, self.connection.password)
                    successful = TRUE
            except all_errors, detail:
                if (type(detail) == ListType):
                    self.log4py.debug("[ %s (Debug) ] FTP login failed: %s - %s" % (str(self.title), detail[0], detail[1]))
                else:
                    self.log4py.debug("[ %s (Debug) ] FTP login failed: %s" % (str(self.title), detail))
                nrretries = nrretries + 1
                sleep(self.delay)
                errormessage = detail

        if (successful != TRUE):
            self.log4py.warn("[ %s ] FTP login failed after %d tries (%s)" % (str(self.title), self.retries, str(errormessage)))
            self.AppendLog([LMError, "Couldn't login to \"%s\" after %d tries (%s)" % (self.connection.hostname, self.retries, str(errormessage))])
            self.notloggedin = TRUE
        else:
            if (self.passivemode == TRUE):
                self.log4py.info("[ %s ] FTP: Setting passive mode" % str(self.title))
                self.ftp.set_pasv(1)
            else:
                self.ftp.set_pasv(0)
            errormessage = None
            self.log4py.info("[ %s ] FTP login succeeded" % str(self.title))
        return errormessage

    # Disconnect from a FTP-site
    def DisconnectFTP(self):
        try:
            self.ftp.quit()
        except:
            pass                                # no need to worry when disconnection failed

    # Reconnect from/to a FTP-site
    def ReconnectFTP(self):
        try:
            self.ftp.quit()
            self.log4py.info("[ %s ] FTP Reconnect: disconnect succeeded" % str(self.title))
        except:
            self.log4py.info("[ %s ] FTP Reconnect: disconnect failed" % str(self.title))
        result = self.ConnectFTP()
        if (result != None):
            self.log4py.error("[ %s ] FTP Reconnect: failed" % str(self.title))
            exit(1)

        self.filelist.insert(0, self.currentfile)

    # Close a connection (ftp or http)
    def CloseConnection(self):
        if (self.connection != None) and (self.connection.status == CSConnected):
            if (self.connection.protocol == CTFTP):
                self.DisconnectFTP()
            elif (self.connection.protocol == CTHTTP):
                self.DisconnectHTTP()
            self.connection.status = CSDisconnected
                
    # Open a connection (ftp or http)
    def OpenConnection(self):
        if (self.connection.protocol == CTFTP):
            result = self.ConnectFTP()
        elif (self.connection.protocol == CTHTTP):
            result = self.ConnectHTTP()
        if (result == None):
            self.connection.status = CSConnected
        return result

    # Callback for downloading a file
    def RetrCallBack(self, line):
        self.downloadfile.write(line)

    # Get the modification time of a remote file
    def GetRemoteMDTM(self, file):
        if (file.modtime == 0):
            try:
                modtime = split(self.ftp.voidcmd("MDTM " + file.path + file.name), " ")[1]
                if (modtime[0:3] == "191"):                 # Fix for old WU-FTP Daemons (y2k bug)
                    modtime = "2000%s" % (modtime[5:])
                year = atoi(modtime[0:4])
                month = atoi(modtime[4:6])
                day = atoi(modtime[6:8])
                hours = atoi(modtime[8:10])
                minutes = atoi(modtime[10:12])
                seconds = atoi(modtime[12:14])
                file.modtime = int(mktime((year, month, day, hours, minutes, seconds, 0, 0, -1)))
            except:
                file.modtime = -1

    # Returns the download filename for a given filename
    def TempFilename(self, filename):
        return "%s/.%s.tmp" % (os.path.dirname(filename), os.path.basename(filename))

    # Checks wether a filename is a tempfile or not
    def IsTempFilename(self, filename):
        tmpfilename = os.path.basename(filename)
        if len(tmpfilename) >= 5:
            if (tmpfilename[0] == ".") and (tmpfilename[-4:] == ".tmp"):
                return TRUE
            else:
                return FALSE
        else:
            return FALSE

    # Returns the real filename of a tempfile
    def RealFilename(self, filename):
        if len(filename) >= 5:
            return filename[1:-4]
        else:
            return filename
        
    # The real file-downloading function (FTP)
    def DownloadFileFTP(self, localfilepath, filename):
        rest = None
        tmpfilename = self.TempFilename(filename)
        if os.path.exists(tmpfilename):
            if (self.continueftp == TRUE):
                size = lstat(tmpfilename)[6]                          # determine file size
                if (self.test == FALSE):
                    self.log4py.info("[ %s ] Restarting at position %.0f (FTP): %s%s" % (str(self.title), size, self.currentfile.path, self.currentfile.name))
                    self.downloadfile = open(tmpfilename, "a")
                    rest = size
                else:
                    self.log4py.info("[ %s (Test) ] Restarting at position %.0f (FTP): %s%s" % (str(self.title), size, self.currentfile.path, self.currentfile.name))
            else:
                if (self.test == FALSE):
                    unlink(tmpfilename)
                    self.downloadfile = open(tmpfilename, "w")
        else:
            if (self.test == FALSE):
                self.downloadfile = open(tmpfilename, "w")

        if (self.test == FALSE):
            self.ftp.retrbinary("RETR %s%s" % (self.currentfile.path, self.currentfile.name), self.RetrCallBack, 1024, rest)
            self.downloadfile.close()
            rename(tmpfilename, filename)
            if (self.owner != None):
                chown(filename, self.owner, self.group)
            if (self.currentfile.modtime != -1):
                utime(localfilepath + self.currentfile.name, (self.currentfile.modtime, self.currentfile.modtime))
            if (self.umask == None):
                chmod(localfilepath + self.currentfile.name, self.currentfile.permissions)

        self.AppendLog([LMRetrieve, "%s%s" % (self.currentfile.path, self.currentfile.name)])

        self.nrbytesretr = self.nrbytesretr + self.currentfile.size
        self.nrfilesretr = self.nrfilesretr + 1
        self.installer.checkfile(self.currentfile.path, self.currentfile.name, filename)
    
    # Download a file
    def DownloadFile(self):
        localfilepath = self.GetLCwd()
        downloadfile = FALSE
        setmdtm = FALSE

        if (self.currentfile.modtime == 0):
            self.GetRemoteMDTM(self.currentfile)
        if (not (os.path.exists(localfilepath + self.currentfile.name))):
            downloadfile = TRUE
        else:
            localinfo = lstat(localfilepath + self.currentfile.name)
            if ((self.currentfile.modtime != int(localinfo[8]) and (self.currentfile.modtime != -1) and (self.ignoremdtm == FALSE))) or (self.currentfile.size != localinfo[6]):
                rm(localfilepath + self.currentfile.name)
                downloadfile = TRUE
            elif ((self.ignoremdtm == TRUE) and (self.currentfile.modtime != -1) and (self.currentfile.modtime != int(localinfo[8]))):
                setmdtm = TRUE

        # If local files older than maxage are been deleted then there's no need to download those
        if (downloadfile == TRUE) and (self.currentfile.modtime != -1) and (self.maxage != 0):
            maxmdtm = long(time() - (self.maxage * 60 * 60 * 24))
            if (self.currentfile.modtime < maxmdtm):
                downloadfile = FALSE

        # Don't download files newer than maxdays
        if (downloadfile == TRUE) and (self.currentfile.modtime != -1) and (self.maxdays != 0):
            newmdtm = long(time() - (self.maxdays * 60 * 60 * 24))
            if (self.currentfile.modtime < newmdtm):
                downloadfile = FALSE

        if (downloadfile == TRUE):
            if (self.test == FALSE):
                self.log4py.info("[ %s ] Downloading (FTP): %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
            else:
                self.log4py.info("[ %s (Test) ] Downloading (FTP): %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))                   
            filename = "%s%s" % (self.GetLCwd(), self.currentfile.name)
            if (self.noexception == TRUE):
                self.DownloadFileFTP(localfilepath, filename)
            else:
                try:
                    self.DownloadFileFTP(localfilepath, filename)
                except all_errors, detail:
                    self.HandleError(detail, ESDownloadFile)
        elif ((setmdtm == TRUE) and (self.currentfile.modtime != -1)):
            if (self.test == FALSE):
                self.log4py.info("[ %s ] Setting MDTM (FTP): %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
            else:
                self.log4py.info("[ %s (Test)] Setting MDTM (FTP): %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
            try:
                if (self.test == FALSE):
                    utime(localfilepath + self.currentfile.name, (self.currentfile.modtime, self.currentfile.modtime))
            except all_errors, detail:
                self.HandleError(detail, ESDownloadFile)
            
    # Optimizes the remote filelist, so that only the latest version will be retrieved
    def ReduceRemoteFileList(self):
        allremotefiles = []

        # Check the remote filelist for files matching each only-latest regexp
        for i in range(len(self.onlylatest)):
            regexpremotefiles = []
            for j in range(len(self.remotefiles)):
                if ((self.onlylatest[i].match(self.remotefiles[j].name) != None) or (self.onlylatest[i].match(self.remotefiles[j].path + self.remotefiles[j].name))) and (self.remotefiles[j].filetype != FTDirectory):
                    regexpremotefiles.append(self.remotefiles[j])
            if (len(regexpremotefiles) > 1):
                mostrecentfile = File()
                for j in range(len(regexpremotefiles)):
                    self.GetRemoteMDTM(regexpremotefiles[j])
                    if (regexpremotefiles[j].modtime > mostrecentfile.modtime):
                        mostrecentfile = regexpremotefiles[j]
                allremotefiles.append(mostrecentfile)
            elif (len(regexpremotefiles) == 1):
                allremotefiles.append(regexpremotefiles[0])

        # Append all directories and files which do not match _any_ only-latest regexp
        for i in range(len(self.remotefiles)):
            if (self.remotefiles[i].filetype == FTDirectory):
                allremotefiles.append(self.remotefiles[i])
            else:
                append = TRUE
                for j in range(len(self.onlylatest)):
                    if ((self.onlylatest[j].match(self.remotefiles[i].name)) != None) or (self.onlylatest[j].match(self.remotefiles[i].path + self.remotefiles[i].name) != None):
                        append = FALSE
                if (append == TRUE):
                    allremotefiles.append(self.remotefiles[i])

        self.remotefiles = allremotefiles
        
    # Get the filetype of the first file (the one specified on the command line)
    def GetFileType(self, cache = TRUE, exclude = TRUE, firstURL = FALSE):
        self.currentfile.filetype = None
        if (self.currentfile.name == "") and (self.currentfile.path == "/"):
            self.currentfile.filetype = FTDirectory
        else:
            if (self.currentfile.name == ""):
                if (self.currentfile.path[-1] == "/") and (firstURL == TRUE):
                    try:
                        self.ftp.cwd(self.currentfile.path)
                        self.currentfile.filetype = FTDirectory
                    except:
                        directory = self.currentfile.path[:rfind(self.currentfile.path, "/", 0, -2)] + "/"
                else:
                        directory = self.currentfile.path[:rfind(self.currentfile.path, "/", 0, -2)] + "/"
            else:
                directory = self.currentfile.path
            if (self.currentfile.filetype == None):
                result = self.GetRemoteFileList(directory, cache, exclude)
                if (result == None):
                    for i in range(len(self.remotefiles)):
                        if ((self.remotefiles[i].path + self.remotefiles[i].name) == (self.currentfile.path + self.currentfile.name)):
                            self.currentfile = copy.copy(self.remotefiles[i])
                    # This second check is necessary, because if the path = /pub2/ and pub2 is a link, it's not found
                    # with the first check !
                    if (self.currentfile.filetype == None) or (self.currentfile.filetype == FTLinkTarget):
                        for i in range(len(self.remotefiles)):
                            if ((self.remotefiles[i].path + self.remotefiles[i].name) + "/" == (self.currentfile.path + self.currentfile.name)):
                                self.currentfile = copy.copy(self.remotefiles[i])
                    # The third check is, if /dir/dir is a directory (no trailing /)
                    if (self.currentfile.filetype == None) or (self.currentfile.filetype == FTLinkTarget):
                        for i in range(len(self.remotefiles)):
                            if ((self.remotefiles[i].path + self.remotefiles[i].name) == (self.currentfile.path + self.currentfile.name + "/")):
                                self.currentfile = copy.copy(self.remotefiles[i])
                else:
                    return result

    # The main mirror loop
    def MainLoop(self):
        while len(self.filelist) > 0:
            self.currentfile = self.filelist[0]
            self.filelist.remove(self.currentfile)
            # Host has changed - open a new connection
            if (self.currentfile.connection != self.connection):
                self.CloseConnection()
                self.connection = self.currentfile.connection
                result = self.OpenConnection()
                if (result == None):
                    self.log4py.debug("[ %s (debug) ] Getting file type of %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
                    localresult = self.GetFileType(cache = FALSE, exclude = FALSE, firstURL = TRUE)        # Get the filetype of the first URL
                    while (self.currentfile.filetype == FTLink):
                        self.log4py.info("[ %s ] Following initial link %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
                        self.currentfile.name, self.currentfile.path = self.AdoptLinkTarget(self.currentfile.name, self.currentfile.path, self.currentfile.linktarget)
                        self.path = self.currentfile.path + self.currentfile.name
                        self.remotefiles = []
                        self.log4py.debug("[ %s (debug) ] Getting file type of %s%s" % (str(self.title), self.currentfile.path, self.currentfile.name))
                        localresult = self.GetFileType(cache = FALSE, exclude = FALSE)
                    self.CheckDirectory(self.GetLCwd())
            if (result == None):
                if (self.currentfile.connection.protocol == CTFTP):
                    if (self.currentfile.filetype == FTRegularFile):
                        self.DownloadFile()
                    elif (self.currentfile.filetype == FTDirectory):
                        localresult = self.GetRemoteFileList()
                        if (localresult == None):
                            if (len(self.onlylatest) > 0):
                                self.ReduceRemoteFileList()
                            self.CheckDirectory(self.GetLCwd())
                            self.GetLocalFileList()
                            self.DelOldLocalFiles()
                            self.GetRemoteFiles()
                    elif (self.currentfile.filetype == FTLinkTarget):
                        self.GetFileType()
                        if self.currentfile.filetype == None:
                            self.GetFileType()
                        self.CheckDirectory(self.GetLCwd())
                        self.filelist.insert(0, self.currentfile)
                    elif (self.currentfile.filetype == FTLink):
                        self.CreateLink()
                    else:
                        self.log4py.info("[ %s ] Warning: %s%s not found" % (str(self.title), self.currentfile.path, self.currentfile.name))
                        self.AppendLog([LMWarning, self.currentfile.path + self.currentfile.name + " not found"])
                elif (self.currentfile.connection.protocol == CTHTTP):
                    self.log4py.warn("[ %s ] Sorry, but ECLiPt-Mirror doesn't support HTTP (yet) - use wget !" % str(self.title))
        self.CloseConnection()

    # Split a URL
    def SplitURL(self):
        file = File()
        file.log4py.set_loglevel(self.log4py.get_loglevel())

        self.protocol, username, password, self.hostname, self.port, self.path, params, query, fragment = urlparse(self.url)
        
        if (self.username == ""):
            self.username = username
        if (self.password == ""):
            self.password = password

        if (lower(self.protocol) == "ftp") and (self.username == ""):
            self.username = DVUsername
            self.password = DVPassword

        file.path, file.name = os.path.split(self.path)
        if (file.path != "/"):
            file.path = file.path + "/"

        return file

    # Check max delete value
    def ConvertMaxDelete(self, path):
        if self.maxdelete != None:
            if (self.maxdelete[-1:] == "%"):
                self.log4py.info("[ %s ] Calculating disk usage ... " % str(self.title))
                if atoi(self.maxdelete[:-1]) != 0:
                    diskusage = du(path)
                    self.maxdelete = divmod(diskusage, atoi(self.maxdelete[:-1]))[0]
                    self.log4py.info("[ %s ] Disk Usage: %.0f Byte ..." % (str(self.title), diskusage))
                else:
                    self.maxdelete = 0
            elif (lower(self.maxdelete[-1:]) == "k"):
                self.maxdelete = atoi(self.maxdelete[:-1]) * 1024
            elif (lower(self.maxdelete[-1:]) == "m"):
                self.maxdelete = atoi(self.maxdelete[:-1]) * 1024 * 1024
        else:
            self.maxdelete = -1

    # Returns the local directory corresponding to FULL, PATH or NONE
    def GetLCwd(self, hostname = "", path = "", parentdirectory = ""):

        if (hostname == ""):
            hostname = self.currentfile.connection.hostname
        if (path == ""):
            path = self.currentfile.path
        if (parentdirectory == ""):
            parentdirectory = self.directory

        if (self.parentformat == PFFull):
            directory = parentdirectory + hostname + path
        elif (self.parentformat == PFPath):
            directory = parentdirectory + path[1:]
        elif (self.parentformat == PFNone):
            shortpath = path[len(self.path):]
            if (len(shortpath) > 0) and (shortpath[0] == "/"):
                shortpath = shortpath[1:]
            directory = parentdirectory + shortpath

        return directory

    # Start the mirror 
    def Start(self):

        self.log4py.info("[ %s ] Starting mirror" % str(self.title))

        file = self.SplitURL()

        self.StartLog()

        connection = Connection()
        connection.protocol = self.protocol
        connection.hostname = self.hostname
        connection.realhostname = self.realhostname
        connection.port = self.port
        connection.username = self.username
        connection.password = self.password
        connection.status = CSDisconnected

        errorcode = 0
        
        if (connection.hostname == ""):
            self.log4py.error("[ %s ] Invalid URL \"%s\" specified - aborting mirror." % (str(self.title), self.url))
            errorcode = 1

        else:
            file.connection = connection

            self.filelist.append(file)
            self.ConvertMaxDelete(self.GetLCwd(file.connection.hostname, self.path))

            if (self.umask != None):
                umask(self.umask)
                self.log4py.info("[ %s ] Setting umask to %s" % (str(self.title), zfill(oct(self.umask), 3)))

            self.MainLoop()
            
        self.End(errorcode)

    # Stop mirror, write the log and terminate
    def End(self, level):
        self.installer.finish()
        self.WriteLog()

        self.log4py.info("[ %s ] Finishing Mirror" % str(self.title))

        sys.exit(level)

# ----------------------------------------------------------------------
# Definition of the Indexfile-class

class IndexFile:	

    def __init__(self):
        self.directory = None               # Directory where to find the logs
        self.mirrorlogs = []                # List of Logs-files
        self.mirrorstatus = {}              # Status of mirrors
        self.mirrorcategory = {}            # Category of the mirror
        self.categorymirror = {}            # Inverse of mirrorcategory
        self.description = {}               # Description of mirrors
        self.timestamp = {}                 # List of time-stamps for mirrors
        self.logfilename = {}               # Name of the mirror logfile
        self.lastchanged = {}               # Last change of the mirror
        self.log4py = log4py.Logger().get_instance(self)

        self.errorcolor = "#CC0033"         # Default error color
        self.warningcolor = "#009933"       # Default color for warnings

        self.logtemplate = None             # Filename of the log template-file
        self.summarytemplate = None         # Filename of the summary template-file
        self.template = []
        self.templatecategory = []
        self.templateloop = []
        self.templatesummary = []
        self.templatesummaryloop = []

        self.logfileformat = LFPlain        # Format of the indexfile

        self.mailusers = ""                 # Mail info about to changes to some users
        self.mailusercmd = ""               # Command to send mail to users

        self.nrbytesretr = 0L
        self.nrfilesretr = 0L
        self.nrlinkscreated = 0L
        self.nrdirscreated = 0L
        self.nrbytesdel = 0L
        self.nrfilesdel = 0L
        self.nrlinksdel = 0L
        self.nrdirsdel = 0L
        self.totalseconds = 0L
        self.firststarttime = None
        self.lastendtime = None

        self.proxy = None                   # Proxy host and port
        
    # Analyze logfiles to get Title, Status & Category of mirrors
    def AnalyzeLogs(self):
        currentdir = getcwd()
        chdir(self.directory)
        for i in range(len(self.mirrorlogs)):
            datei = open(self.mirrorlogs[i], "r")
            title = ""
            line = "<empty>"
            copyfile = FALSE
            while line != "":
                line = datei.readline()
                stripped = strip(line)
                if (stripped[0:4] == "<!--"):
                    restofline = strip(sub("-->", "", stripped[5:]))
                    if restofline[0:5] == "TITLE":
                        title = restofline[7:]
                        self.logfilename[title] = self.mirrorlogs[i]
                        self.mirrorstatus[title] = MSunknown
                        self.description[title] = "None"
                        self.timestamp[title] = "0"
                    elif restofline[0:6] == "STATUS":
                        self.mirrorstatus[title] = restofline[8:]
                        if restofline[8:][0] in digits:
                            self.lastchanged[title] = restofline[8:]
                            copyfile = TRUE
                    elif restofline[0:8] == "CATEGORY":
                        category = restofline[10:]
                        self.categorymirror[title] = category
                        if self.mirrorcategory.has_key(category):
                            self.mirrorcategory[category].append(title)
                        else:
                            self.mirrorcategory[category] = [title]
                    elif restofline[0:9] == "TIMESTAMP":
                        self.timestamp[title] = restofline[11:]
                    elif restofline[0:11] == "DESCRIPTION":
                        self.description[title] = restofline[13:]
                    elif restofline[0:10] == "STATISTICS":
                        statsline = strip(restofline[12:])
                        splitted = split(statsline)

                        for j in range(len(splitted)):
                            splitted[j] = atol(splitted[j])

                        currentday = localtime(time())
                        midnight = mktime((currentday[0], currentday[1], currentday[2], 0, 0, 0, 0, 0, currentday[8]))

                        if (splitted[8] >= midnight):

                            self.nrbytesretr = self.nrbytesretr + splitted[0]
                            self.nrfilesretr = self.nrfilesretr + splitted[1]
                            self.nrlinkscreated = self.nrlinkscreated + splitted[2]
                            self.nrdirscreated = self.nrdirscreated + splitted[3]
                            self.nrbytesdel = self.nrbytesdel + splitted[4]
                            self.nrfilesdel = self.nrfilesdel + splitted[5]
                            self.nrlinksdel = self.nrlinksdel + splitted[6]
                            self.nrdirsdel = self.nrdirsdel + splitted[7]

                            self.totalseconds = self.totalseconds + (splitted[9] - splitted[8])

                            if (self.firststarttime == None) or (self.firststarttime > splitted[8]):
                                self.firststarttime = splitted[8]
                            if (self.lastendtime == None) or (self.lastendtime < splitted[9]):
                                self.lastendtime = splitted[9]

            datei.close()
            if (copyfile == TRUE): # save last logfile, where changes have occured
                allfiles = listdir(".")
                for j in range(len(allfiles)):
                    if (match(".*" + self.mirrorlogs[i], allfiles[j]) != None):
                        if (allfiles[j][0] in digits):
                            try:
                                unlink(allfiles[j])
                            except:
                                self.log4py.error("[ main ] Index-file: Error while unlinking \"%s\" - Please check the file / directory permissions !" % allfiles[j])
                try:
                    shutil.copy(self.mirrorlogs[i], self.mirrorstatus[title] + "-" + self.mirrorlogs[i])
                except:
                    self.log4py.error("[ main ] Index-file: Error while copying \"%s -> %s\" - Please check the file / directory permissions !" % (self.mirrorlogs[i], self.mirrorstatus[title] + "-" + self.mirrorlogs[i]))
                    
        chdir(currentdir)

    # Read Template file
    def ReadTemplateFile(self):
        file = open(self.logtemplate)
        line = "<empty>"
        self.template = []
        self.templatecategory = []
        self.templateloop = []
        target = self.template
        while (line != ""):
            line = file.readline()
            if (lower(strip(line)) == "$categorystart$"):
                target.append("$categoryloop$")
                target = self.templatecategory
            elif (lower(strip(line)) == "$categoryend$"):
                target = self.template
            elif (lower(strip(line)) == "$mirrorstart$"):
                target.append("$mirrorloop$")
                target = self.templateloop
            elif (lower(strip(line)) == "$mirrorend$"):
                target = self.templatecategory
            elif (find(strip(lower(line)), "<!-- errorcolor:") != -1):
                color = strip(lower(line))
                position = find(color, "<!-- errorcolor:")
                color = color[position + len("<!-- errorcolor:"):]
                position = find(color, "-->")
                color = strip(color[:position])
                self.errorcolor = color
            elif (find(strip(lower(line)), "<!-- warningcolor:") != -1):
                color = strip(lower(line))
                position = find(color, "<!-- warningcolor:")
                color = color[position + len("<!-- warningcolor:"):]
                position = find(color, "-->")
                color = strip(color[:position])
                self.warningcolor = color
            else:
                target.append(line)
        file.close()

    # Read Template file
    def ReadSummaryTemplateFile(self, datei, fileupdated, fileerrors):
        file = open(self.summarytemplate)
        line = "<empty>"
        self.summarytemplate = []
        self.summarytemplateloop = []
        target = self.summarytemplate
        while (line != ""):
            line = file.readline()
            line = sub("\$indexfilename\$", datei.name, line)
            line = sub("\$updatedfilename\$", fileupdated.name, line)
            line = sub("\$errorfilename\$", fileerrors.name, line)
            if (lower(strip(line)) == "$mirrorstart$"):
                target.append("$mirrorloop$")
                target = self.summarytemplateloop
            elif (lower(strip(line)) == "$mirrorend$"):
                target = self.summarytemplate
            else:
                target.append(line)
        file.close()

    # Create the master index-file
    def CreateMasterIndexFile(self, categorylist, duration, downloaded, deleted, downloadrate, datei, categorykeys, fileupdated, fileerrors):
        for i in range(len(self.template)):
            line = self.template[i]
            if (line == "$categoryloop$"):                           # Add lines for each category
                for j in range(len(categorykeys)):
                    category = categorykeys[j]
                    mirrorlist = self.mirrorcategory[categorykeys[j]]
                    mirrorlist.sort()
                    for k in range(len(self.templatecategory)):
                        line = self.templatecategory[k]
                        if (line == "$mirrorloop$"):                 # Add lines for each mirror
                            for l in range(len(mirrorlist)):
                                title = mirrorlist[l]
                                update = "Unknown"
                                if (self.logfileformat == LFPlain):
                                    mirrortitle = title
                                    if self.mirrorstatus[title] == MSunknown: status = "in progress"
                                    elif self.mirrorstatus[title] == MSnotloggedin: status = "not mirrored"
                                    elif self.mirrorstatus[title] == MSunchanged: status = "unchanged"
                                    else: status = "updated"
                                    if self.lastchanged.has_key(title):
                                        update = ctime(atof(self.lastchanged[title]))
                                else:
                                    mirrortitle = "<a href=\"%s\">%s</a>" % (self.logfilename[title], title)
                                    if self.mirrorstatus[title] == MSunknown: status = "<font color=\"%s\">in progress</font>" % self.warningcolor
                                    elif self.mirrorstatus[title] == MSnotloggedin: status = "<font color=\"%s\">not mirrored</font>" % self.errorcolor
                                    elif self.mirrorstatus[title] == MSunchanged: status = "unchanged"
                                    else: status = "<font color=\"%s\">updated</font>" % self.warningcolor
                                    if self.lastchanged.has_key(title):
                                        update = "<a href=\"%s\">%s</a>" % (self.lastchanged[title] + "-" + self.logfilename[title], ctime(atof(self.lastchanged[title])))
                                for m in range(len(self.templateloop)):
                                    line = self.templateloop[m]
                                    line = sub("\$status\$", status, line)
                                    line = sub("\$title\$", mirrortitle, line)
                                    line = sub("\$update\$", update, line)
                                    datei.write(line)
                        else:
                            line = sub("\$category\$", category, line)
                            datei.write(line)
            else:
                line = sub("\$categorylist\$", categorylist, line)
                line = sub("\$lastupdate\$", ctime(time()), line)
                if (self.firststarttime != None):
		    line = sub("\$starttime\$", ctime(self.firststarttime), line)
		else:
		    line = sub("\$starttime\$", "Unknown", line)
		if (self.lastendtime != None):
		    line = sub("\$endtime\$", ctime(self.lastendtime), line)
		else:
                    line = sub("\$endtime\$", "Unknown", line)

                line = sub("\$duration\$", duration, line)
                line = sub("\$downloadstats\$", downloaded, line)
                line = sub("\$deletestats\$", deleted, line)
                line = sub("\$downloadrate\$", downloadrate, line)
                line = sub("\$proxy\$", str(self.proxy), line)
                line = sub("\$indexfilename\$", datei.name, line)
                line = sub("\$updatedfilename\$", fileupdated.name, line)
                line = sub("\$errorfilename\$", fileerrors.name, line)
                datei.write(line)

    # Create errors & updated file
    def CreateSummaryFile(self, fileupdated, fileerrors):

        updatedlist = {}
        errorlist = {}
        for i in range(len(self.mirrorcategory.keys())):
            category = self.mirrorcategory.keys()[i]
            for j in range(len(self.mirrorcategory[category])):
                title = self.mirrorcategory[category][j]
                timestamp = self.timestamp[title]
                description = self.description[title]
                if self.mirrorstatus[title] == MSnotloggedin:
                    errorlist[timestamp] = title
                elif (self.mirrorstatus[title] != MSunchanged) and (self.mirrorstatus[title] != MSunknown):
                    updatedlist[timestamp] = title
                
        for i in range(len(self.summarytemplate)):
            line = self.summarytemplate[i]
            if (line == "$mirrorloop$"):
                
                # Write updated file information

                if len(updatedlist.keys()) == 0:
                    updatedlist["0"] = "None"
                    self.categorymirror["None"] = "None"
                    self.logfilename["None"] = ""
                    self.description["None"] = "No packages have been updated"
                    
                timelist = updatedlist.keys()
                timelist.sort()

                for j in range(len(timelist)):
                    timestamp = timelist[j]
                    title = updatedlist[timestamp]
                    mirrortitle = title
                    categorylink = self.categorymirror[title]
                    if (self.logfileformat == LFHTML) or (self.logfileformat == LFPHP):
                        mirrortitle = "<a href=\"%s\">%s</a>" % (self.logfilename[title], title)
                        if (self.logfileformat == LFHTML):
                            categorylink = "<a href=\"%s\">%s</a>" % ("index" + LEHTML, self.categorymirror[title])
                        elif (self.logfileformat == LFPHP):
                            categorylink = "<a href=\"%s#%s\">%s</a>" % ("index" + LEPHP, self.categorymirror[title], self.categorymirror[title])
                            
                    for k in range(len(self.summarytemplateloop)):
                        line = self.summarytemplateloop[k]
                        line = sub("\$update\$", asctime(localtime(atof(timestamp))), line)
                        line = sub("\$title\$", mirrortitle, line)
                        line = sub("\$category\$", categorylink, line)
                        line = sub("\$description\$", self.description[title], line)
                        fileupdated.write(line)

                # Write error file information

                if len(errorlist.keys()) == 0:
                    errorlist["0"] = "None"
                    self.categorymirror["None"] = "None"
                    self.logfilename["None"] = ""
                    self.description["None"] = "No errors have occured"

                timelist = errorlist.keys()
                timelist.sort()
                for j in range(len(timelist)):
                    timestamp = timelist[j]
                    title = errorlist[timestamp]
                    mirrortitle = title
                    categorylink = self.categorymirror[title]
                    if (self.logfileformat == LFHTML) or (self.logfileformat == LFPHP):
                        mirrortitle = "<a href=\"%s\">%s</a>" % (self.logfilename[title], title)
                        if (self.logfileformat == LFHTML):
                            categorylink = "<a href=\"%s\">%s</a>" % ("index" + LEHTML, self.categorymirror[title])
                        elif (self.logfileformat == LFPHP):
                            categorylink = "<a href=\"%s#%s\">%s</a>" % ("index" + LEPHP, self.categorymirror[title], self.categorymirror[title])

                    for k in range(len(self.summarytemplateloop)):
                        line = self.summarytemplateloop[k]
                        line = sub("\$update\$", asctime(localtime(atof(timestamp))), line)
                        line = sub("\$title\$", mirrortitle, line)
                        line = sub("\$category\$", categorylink, line)
                        line = sub("\$description\$", self.description[title], line)
                        fileerrors.write(line)

            else:
                line = sub("\$lastupdate\$", ctime(time()), line)
                if find(line, "$summary$") != -1:
                    lineupdated = sub("\$summary\$", "Updated mirrors", line)
                    lineerrors = sub("\$summary\$", "Failed mirrors", line)
                    fileupdated.write(lineupdated)
                    fileerrors.write(lineerrors)
                else:
                    fileupdated.write(line)
                    fileerrors.write(line)

    # Create the new Index-file
    def CreateFile(self):
        currentdir = getcwd()
        chdir(self.directory)

        if (self.logtemplate == None) or (self.summarytemplate == None):
            self.log4py.error("[ Index ] Logfile: Error: No template file specified !")
            return

        try:
            if (self.logfileformat == LFHTML):
                datei = open("index" + LEHTML, "w")
                fileupdated = open("updated" + LEHTML, "w")
                fileerrors = open("errors" + LEHTML, "w")
            elif (self.logfileformat == LFPHP):
                datei = open("index" + LEPHP, "w")
                fileupdated = open("updated" + LEPHP, "w")
                fileerrors = open("errors" + LEPHP, "w")
            elif (self.logfileformat == LFPlain):
                datei = open("index" + LEPlain, "w")
                fileupdated = open("updated" + LEPlain, "w")
                fileerrors = open("errors" + LEPlain, "w")
        except:
            self.log4py.error("[ Index ] Logfile: Couldn't open index file !")
            return

        self.ReadTemplateFile()
        self.ReadSummaryTemplateFile(datei, fileupdated, fileerrors)

        # Calculate the time
        minutes, seconds = divmod(self.totalseconds, 60)
        hours, minutes = divmod(minutes, 60)

        # Create Format specific text
        if (self.logfileformat == LFPlain):
            duration = "%2d Hour(s), %2d Minute(s) and %2d Second(s)" % (hours, minutes, seconds)
            downloaded = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
            deleted = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)
        else:
            duration = "<b>%2d</b> Hour(s), <b>%2d</b> Minute(s) and <b>%2d</b> Second(s)" % (hours, minutes, seconds)
            downloaded = "<b>%s</b> Byte(s) in <b>%.0f</b> File(s), <b>%.0f</b> Link(s) and <b>%.0f</b> Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
            deleted = "<b>%s</b> Byte(s) in <b>%.0f</b> File(s), <b>%.0f</b> Link(s) and <b>%.0f</b> Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)

        # Calculate download rate
        if (self.nrbytesretr > 0):
            if (self.totalseconds > 0):
                rate = self.nrbytesretr / self.totalseconds
            else:
                rate = 0
            kbytepersec = int(rate / 1024)
            if (kbytepersec > 0):
                if (self.logfileformat == LFPlain):
                    downloadrate = "%.0f KByte/sec" % int(kbytepersec)
                else:
                    downloadrate = "<b>%.0f</b> KByte/sec" % int(kbytepersec)
            else:
                if (self.logfileformat == LFPlain):
                    downloadrate = "%.0f Byte/sec" % int(rate)
                else:
                    downloadrate = "<b>%.0f</b> Byte/sec" % int(rate)
        else:
            downloadrate = "Nothing downloaded"

        categorykeys = self.mirrorcategory.keys()
        categorykeys.sort()

        categorylist = ""

        for i in range(len(categorykeys)):
            if (self.logfileformat == LFPlain):
                categorylist = "%s%s" % (categorylist, categorykeys[i])
            else:
                categorylist = "%s<a href=\"#%s\">%s</a>" % (categorylist, categorykeys[i], categorykeys[i])
            if (i < (len(categorykeys) - 1)):
                categorylist = "%s | " % categorylist

        self.CreateMasterIndexFile(categorylist, duration, downloaded, deleted, downloadrate, datei, categorykeys, fileupdated, fileerrors)
        self.CreateSummaryFile(fileupdated, fileerrors)

        datei.close()
        fileupdated.close()
        fileerrors.close()

        chdir(currentdir)

    def GetLastlog(self):
        currentdir = getcwd()
        chdir(self.directory)
        if os.path.exists(LastlogFilename):
            datei = open(LastlogFilename, "r")
            line = "<empty>"
            while line != "":
                line = datei.readline()
                if strip(line) != "":
                    splitted = split(strip(line), ";")
                    if not self.lastchanged.has_key(splitted[0]):
                        self.lastchanged[splitted[0]] = splitted[1]
            datei.close()
        chdir(currentdir)

    def WriteLastlog(self):
        currentdir = getcwd()
        chdir(self.directory)
        try:
            datei = open(LastlogFilename, "w")
        except:
            self.log4py.error("[ Index ] Logfile: Couldn't write lastlog file !")
            return
        alltitles = self.lastchanged.keys()
        for i in range(len(alltitles)):
            datei.write(alltitles[i] + ";" + self.lastchanged[alltitles[i]] + "\n")
        datei.close()
        chdir(currentdir)

    # Send mail to users, if things have changed or if emirror couldn't log in
    def MailUsers(self):

        changed = []
        notloggedin = []

        for i in range(len(self.mirrorstatus.keys())):
            key = self.mirrorstatus.keys()[i]
            if (self.mirrorstatus[key] == MSnotloggedin):
                notloggedin.append(key)
            elif ((self.mirrorstatus[key] != MSunknown) and (self.mirrorstatus[key] != MSunchanged)):
                changed.append(key)

        changed.sort()
        notloggedin.sort()

        if (len(changed) > 0) or (len(notloggedin) > 0):
            self.log4py.info("[ Main ] Mirrors have changed / updated - sending mail.")

            parser = ConfigParser()
            parser.read(self.mailusers)
            emailaddresses = split(parser.get("email", "list"), ",")

            filename = mktemp()
            file = open(filename, "w")
            file.write("\nWelcome to the EMirror EMail-Update Service !\n\n")

            # Calculate the time
            minutes, seconds = divmod(self.totalseconds, 60)
            hours, minutes = divmod(minutes, 60)

            # Create Format specific text
            duration = "%2d Hour(s), %2d Minute(s) and %2d Second(s)" % (hours, minutes, seconds)
            downloaded = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesretr), self.nrfilesretr, self.nrlinkscreated, self.nrdirscreated)
            deleted = "%s Byte(s) in %.0f File(s), %.0f Link(s) and %.0f Directories ..." % (lng2str(self.nrbytesdel), self.nrfilesdel, self.nrlinksdel, self.nrdirsdel)

            if (self.nrbytesretr > 0):
                if (self.totalseconds > 0):
                    rate = self.nrbytesretr / self.totalseconds
                else:
                    rate = 0
                kbytepersec = int(rate / 1024)
                if (kbytepersec > 0):
                    downloadrate = "%.0f KByte/sec" % int(kbytepersec)
                else:
                    downloadrate = "%.0f Byte/sec" % int(rate)
            else:
                downloadrate = "Nothing downloaded"
    
            if (self.firststarttime != None):
                file.write("Start of first mirror: %s\n" % ctime(self.firststarttime))
            else:
                file.write("Start of first mirror: <not set>\n")
            if (self.lastendtime != None):
                file.write("   End of last mirror: %s\n" % ctime(self.lastendtime))
            else:
                file.write("   End of last mirror: <not set>\n")
            file.write("       Total duration: %s\n" % duration)
            file.write("           Downloaded: %s\n" % downloaded)
            file.write("   Avg. download rate: %s\n" % downloadrate)
            file.write("              Deleted: %s\n" % deleted)
            file.write("      Proxy host/port: %s\n\n" % str(self.proxy))

            file.write("The following list of mirrors has changed:\n\n")
            for i in range(len(changed)):
                file.write("    %s [%s]\n" % (changed[i], self.categorymirror[changed[i]]))
            if (len(changed) == 0):
                file.write("    [nothing has changed]\n")
            file.write("\nEMirror couldn't log in into those mirrors. This probably doesn't mean\nanything, but if it happens more often, you maybe have to check that !\n\n")
            for i in range(len(notloggedin)):
                file.write("    %s [%s]\n" % (notloggedin[i], self.categorymirror[notloggedin[i]]))
            if (len(notloggedin) == 0):
                file.write("    [none]\n")
            file.close()
            for i in range(len(emailaddresses)):
                emailaddress = emailaddresses[i]
                self.log4py.debug("[ Main (debug) ] Executing mail-command: cat %s | %s %s " % (filename, self.mailusercmd, emailaddress))
                system("cat %s | %s %s" % (filename, self.mailusercmd, emailaddress))

            unlink(filename)

    def Start(self):
        if (self.directory[-1:] != "/"):
            self.directory = self.directory + "/"
        if not os.path.exists(self.directory):
            self.log4py.error("[ Main ] Error: Directory \"%s\" doesn't exist !" % self.directory)
            exit(1)

        allfiles = listdir(self.directory)

        excludefiles = compile("index.*|.*jpg$|.*gif$")
        for i in range(len(allfiles)):
            if (allfiles[i][0] not in digits) and (not excludefiles.match(allfiles[i])):
                self.mirrorlogs.append(allfiles[i])

        self.AnalyzeLogs()
        self.GetLastlog()
        self.CreateFile()
        self.WriteLastlog()
        if (self.mailusers != "") and (self.mailusercmd != ""):
            self.MailUsers()
        else:
            if (self.mailusers != "") or (self.mailusercmd != ""):
                self.log4py.warn("[ Main ] Warning: you have to set mail-users AND mail-user-cmd for sending mail !")

# ----------------------------------------------------------------------
# Definition of the Program-Class:

# A global Variable for the PIDs of concurrent children
children = []

class Program:

    # Initialisation of variables
    def __init__(self):
        self.mirror = Mirror()                   # The mirror to set parameters
        self.mirrorlist = []                     # List of mirrors
        self.indexfile = IndexFile()             # IndexFile to create
        self.configfile = None                   # Master configuration file
        self.configdirectory = None              # Directory containing configuration files
        self.maxfork = 1                         # How many parallel mirrors shall I run
        self.uid = None                          # The UID of the user who executes the program
        self.log4py = log4py.Logger().get_instance(self)
        self.pidfile = "%s/.emirror.pid" % environ["HOME"]    # Filename of the PID-File
        self.killprocesses = TRUE                # Kill old processes on startup or not

    # Print some information about the program & syntax
    def PrintInfo(self, message, errormsg = None):
        sys.stdout.write("%s\n" % PIHeader)
        if (errormsg != None):
            sys.stdout.write("%s. Try emirror --help for a list of possible parameters.\n" % capitalize(errormsg))
        else:
            sys.stdout.write("%s\n" % message)
            sys.stdout.write("%s\n" % PIOptions)
        exit(0)

    # checks wether a file exists and reports an error if not
    def FileExists(self, file):
        if (not os.path.exists(file)):
            self.log4py.warn("[ Main ] Warning: file \"%s\" does not exist." % file)
            return ""
        else:
            if (find(file, "/") != -1):
                return file
            else:
                return getcwd() + "/" + file

    # Check command line / configuration file options
    def CheckOptions(self, parameter, value, filename = "", option = None):
        if (parameter[0:2] == "--"):
            parameter = parameter[2:]

        # Startup
        if (parameter == "-V" or parameter == "version"):
            sys.stdout.write("%s\n" % PIHeader)
            sys.stdout.write("%s\n" % PIRevLog)
            exit(0)
        elif (parameter == "-h" or parameter == "help"):
            self.PrintInfo(PIUsage)
        elif (parameter == "-v" or parameter == "verbose"):
            self.log4py.set_loglevel(log4py.LOGLEVEL_VERBOSE)
            self.mirror.log4py.set_loglevel(log4py.LOGLEVEL_VERBOSE)
            self.mirror.installer.log4py.set_loglevel(log4py.LOGLEVEL_VERBOSE)
            self.indexfile.log4py.set_loglevel(log4py.LOGLEVEL_VERBOSE)

        # Logging and input files
        elif (parameter == "-o" or parameter == "output-file"):
            self.mirror.logfilename = value
        elif (parameter == "-O" or parameter == "output-directory"):
            if value[-1:] != "/":
                value = value + "/"
            self.mirror.logdirectory = value
        elif (parameter == "-f" or parameter == "output-format"):
            format = lower(value)
            if (format != LFHTML) and (format != LFPlain) and (format != LFPHP):
                self.log4py.warn("[ Main ] Warning: invalid logfile format, choose HTML or Plain.")
                format = LFHTML
            self.mirror.logfileformat = format
            self.indexfile.logfileformat = format
        elif (parameter == "-c" or parameter == "config-file"):
            self.configfile = value
        elif (parameter == "-d" or parameter == "config-directory"):
            self.configdirectory = value
        elif (parameter == "-n" or parameter == "index-directory"):
            if value[-1:] != "/":
                value = value + "/"
            self.indexfile.directory = value
        elif (parameter == "category"):
            self.mirror.category = value
        elif (parameter == "description"):
            self.mirror.description = value
        elif (parameter == "title"):
            self.mirror.title = value
            self.mirror.installer.title = value
        elif (parameter == "email"):
            self.mirror.adminemail = value
        elif (parameter == "mail-users"):
            self.indexfile.mailusers = self.FileExists(value)
        elif (parameter == "mail-user-cmd"):
            self.indexfile.mailusercmd = value
        elif (parameter == "log-template"):
            self.mirror.logtemplate = self.FileExists(value)
        elif (parameter == "index-template"):
            self.indexfile.logtemplate = self.FileExists(value)
        elif (parameter == "summary-template"):
            self.indexfile.summarytemplate = self.FileExists(value)

        # Download and FTP options
        elif (parameter == "-P" or parameter == "parent"):
            if value[-1:] != "/":
                value = value + "/"
            if value[0] != "/":
                self.log4py.warn("[ Main ] Warning: use absolut values for the \"parent\" parameter !.")
                value = "%s/%s" % (os.getcwd(), value)
            self.mirror.directory = value
        elif (parameter == "-D" or parameter == "download-url"):
            if value[-1:] != "/":
                value = value + "/"
            self.mirror.localurl = value
        elif (parameter == "-t" or parameter == "tries"):
            self.mirror.retries = atoi(value)
        elif (parameter == "-w" or parameter == "wait"):
            self.mirror.delay = atoi(value)
        elif (parameter == "-U" or parameter == "umask"):
            self.mirror.umask = atoi(value, 8)
        elif (parameter == "-r" or parameter == "recursive"):
            if (str2bool(value) == TRUE) or (value == ""):
                self.mirror.recursive = TRUE
            elif (str2bool(value) == FALSE):
                self.mirror.recursive = FALSE
            else:
                self.log4py.warn("[ Main ] Warning: invalid value for -r/--recursive. Use Yes/No or TRUE/False.")

        elif (parameter == "-e" or parameter == "exclude"):
            try:
                self.mirror.exclude = compile(unfold(value))
            except:
                self.log4py.error("[ Main ] Error in exclude-regexp: %s" % value)
                exit(1)

        elif (parameter == "-i" or parameter == "include"):
            try:
                self.mirror.include = compile(unfold(value))
            except:
                self.log4py.error("[ Main ] Error in include-regexp: %s" % value)
                exit(1)

        elif (parameter == "ignore"):
            try:
                self.mirror.ignore = compile(unfold(value))
            except:
                self.log4py.error("[ Main ] Error in ignore-regexp: %s" % value)
                exit(1)

        elif (parameter == "proxy"):
            splitted = split(value, ":")
            if (len(splitted) != 2):
                self.log4py.warn("[ Main ] Invalid value for \"Proxy\", use <hostname:port> !")
            else:
                self.mirror.proxy = value
                self.indexfile.proxy = value
                if self.mirror.passivemode == FALSE:
                    self.mirror.passivemode = TRUE
                    
        elif (parameter == "test"):
            if (str2bool(value) == TRUE) or (value == ""):
                self.mirror.test = TRUE
            elif (str2bool(value) == FALSE):
                self.mirror.test = FALSE
            else:
                self.log4py.warn("[ Main ] Warning: invalid value for --test. Use Yes/No or True/False.")

        # Set of only-latest values
        elif (parameter == "-l" or parameter == "only-latest"):
            try:
                regexp = compile(unfold(value))
                self.mirror.onlylatest.append(regexp)
            except:
                self.log4py.error("[ Main ] Error in only-latest-regexp: %s" % value)
                exit(1)

        # Set of auto install values
        elif (parameter == "auto-install"):
            self.mirror.installer.add(option, value)

        elif (parameter == "-u" or parameter == "username"):
            self.mirror.username = value
        elif (parameter == "-p" or parameter == "password"):
            self.mirror.password = value
        elif (parameter == "-m" or parameter == "max-parallel"):
            self.maxfork = atoi(value)
        elif (parameter == "-R" or parameter == "real-hostname"):
            self.mirror.realhostname = value
        elif (parameter == "-M" or parameter == "max-days"):
            self.mirror.maxdays = atoi(value)
        elif (parameter == "-A" or parameter == "max-age"):
            self.mirror.maxage = atoi(value)
        elif (parameter == "-E" or parameter == "max-delete"):
            if (value[-1:] != "%") and (lower(value[-1:]) != "k") and (lower(value[-1:]) != "m"):
                self.log4py.warn("[ Main ] Warning: invalid value for --max-delete (has to end in M, K or %).")
            else:
                self.mirror.maxdelete = value
        elif (parameter == "-F" or parameter == "follow-symlinks"):
            if (str2bool(value) == TRUE) or (value == ""):
                self.mirror.followsymlinks = TRUE
            elif (str2bool(value) == FALSE):
                self.mirror.followsymlinks = FALSE
            else:
                self.log4py.warn("[ Main ] Warning: invalid value for -F/--follow-symlinks. Use Yes/No or True/False.")
        elif (parameter == "-s" or parameter == "parent-format"):
            if (lower(value) == "full") or (lower(value) == "path") or (lower(value) == "none"):
                self.mirror.parentformat = lower(value)
            else:
                self.log4py.warn("[ Main ] Warning: invalid value for -s/--parent-format: use FULL, PATH or NONE.")
        elif (parameter == "ignore-mdtm"):
            if (str2bool(value) == TRUE) or (value == ""):
                self.mirror.ignoremdtm = TRUE
            elif (str2bool(value) == FALSE):
                self.mirror.ignoremdtm = FALSE
            else:
                self.log4py.warn("[ Main ] Warning: invalid value for --ignore-mdtm: use Yes/No or True/False.")
        elif (parameter == "owner"):
            if (self.uid != 0):
                self.log4py.warn("[ Main ] Warning: only user \"root\" can set the owner of files.")
            else:
                if (find(value, ":") == -1):
                    self.log4py.warn("[ Main ] Warning: wrong format for owner: use UID:GID")
                else:
                    splitted = split(value, ":")
                    self.mirror.owner = atoi(splitted[0])
                    self.mirror.group = atoi(splitted[1])

        elif (parameter == "delete-local"):
            if (str2bool(value) == TRUE) or (value == ""):
                self.mirror.deletelocal = TRUE
            elif (str2bool(value) == FALSE):
                self.mirror.deletelocal = FALSE
            else:
                self.log4py.warn("[ Main ] Warning: invalid value for --delete-local: use Yes/No or True/False.")
        elif (parameter == "passive"):
            if (str2bool(value) == TRUE) or (value == ""):
                self.mirror.passivemode = TRUE
            elif (str2bool(value) == FALSE):
                self.mirror.passivemode = FALSE
            else:
                self.log4py.warn("[ Main ] Warning: invalid value for --passive: use Yes/No or True/False.")
        elif (parameter == "continue"):
            if (str2bool(value) == TRUE) or (value == ""):
                self.mirror.continueftp = TRUE
            elif (str2bool(value) == FALSE):
                self.mirror.continueftp = FALSE
            else:
                self.log4py.warn("[ Main ] Warning: invalid value for --continue: use Yes/No or True/False.")
                
        # Error handling options
        elif (parameter == "error-mailcmd"):
            self.mirror.errormailcmd = value

        # Info handling
        elif (parameter == "info-mailcmd"):
            self.mirror.infomailcmd = value
            
        # Debug options
        elif (parameter == "debug"):
            self.log4py.set_loglevel(log4py.LOGLEVEL_DEBUG)
            self.mirror.log4py.set_loglevel(log4py.LOGLEVEL_DEBUG)
            self.mirror.installer.log4py.set_loglevel(log4py.LOGLEVEL_DEBUG)
            self.indexfile.log4py.set_loglevel(log4py.LOGLEVEL_DEBUG)

        elif (parameter == "noexception"):
            self.mirror.noexception = TRUE

        # Other options
        elif (parameter == "url"):
            self.mirror.url = value
        elif (parameter == "-k" or parameter == "no-kill"):
            self.killprocesses = FALSE
        elif (parameter == "pid-file"):
            self.pidfile = value

        else:
            log("%s" % PIHeader)
            if (filename != ""):
                self.log4py.error("[ Main ] Error in config-file %s - invalid parameter \"%s\"\n" % (filename, parameter))
            else:
                self.log4py.error("[ Main ] Error - invalid parameter \"%s\"\n\n" % parameter)
            exit(1)

    # Parse the command line, if a url has been specified
    def ParseCmdLine(self):
        if environ.has_key(EMirrorEnvOptions):
            envargv = split(environ[EMirrorEnvOptions])
            fullargv = envargv + argv[1:]
        else:
            fullargv = argv[1:]
        try:
            optlist, args = getopt.getopt(fullargv,
                                          'Vhvo:O:f:c:d:n:P:D:t:w:re:i:lu:p:m:R:M:A:E:Fs:U:k',
                                          ['version', 'help', 'verbose', 'output-file=', 'output-directory=', 'output-format=', 
                                           'config-file=', 'config-directory=', 'index-directory=', 'category=',
                                           'description=', 'title=', 'email=', 'parent=', 'download-url=', 'tries=', 'wait=', 
                                           'recursive', 'exclude=', 'include=', 'only-latest', 'username=', 'password=',
                                           'max-parallel=', 'real-hostname=',
                                           'max-days=', 'max-age=', 'passive', 'umask=',
                                           'max-delete=', 'log-template=', 'index-template=',
                                           'follow-symlinks=', 'parent-format=', 'error-mailcmd=', 'ignore-mdtm', 'owner=', 
                                           'delete-local', 'no-kill', 'pid-file=',
                                           'mail-users=', 'mail-user-cmd=', 'debug', 'noexception', 'proxy=', 'continue', 
                                           'info-mailcmd=', 'test', 'ignore='])
        except getopt.error, detail:
            self.PrintInfo(PIUsage, detail)

        for flag, arg in optlist:
            self.CheckOptions(flag, arg)

        if (self.configfile == None) and (self.indexfile.directory == None):
            if (len(args) != 1):
                self.PrintInfo(PIUsage)
        if (len(args) > 0):
            self.mirror.url = args[0]
            if (lower(self.mirror.url[0:6]) != "ftp://") and (lower(self.mirror.url[0:7]) != "http://"):
                self.log4py.error("%s" % PIHeader)
                self.log4py.error("Error: specified protocol not supported ...\n")
                exit(1)
            self.mirrorlist.append(self.mirror)

    # Read a configuration file
    def ReadConfigFile(self, configfilename, general = None):
        found = FALSE
        if os.path.exists(configfilename):
            found = TRUE
        if (found == FALSE):
            if environ.has_key(EMirrorEnvironment):
                emirrorcfgpath = environ[EMirrorEnvironment]
                if emirrorcfgpath[-1:] != "/":
                    emirrorcfgpath = emirrorcfgpath + "/"
                configfilename = emirrorcfgpath + configfilename
            if not os.path.exists(configfilename):
                self.log4py.error("%s" % PIHeader)
                self.log4py.error("Configuration file \"%s\" does't exist ...\n" % configfilename)
                exit(1)

        if (general == None):
            parser = ConfigParser()
        else:
            parser = general
        try:
            parser.read(configfilename)
        except:
            self.log4py.error("[ Main ] Error: invalid config-file: %s" % configfilename)
            exit(1)

        for i in range(len(parser.sections())):
            section = parser.sections()[i]
            for j in range(len(parser.options(section))):
                option = parser.options(section)[j]
                if (option != "name") and (option != "__name__"):               # name is the name of the section
                    if (lower(section) == "only-latest"):
                        self.CheckOptions("only-latest", parser.get(section, option), configfilename)
                    elif (lower(section) == "auto-install"):
                        self.CheckOptions("auto-install", parser.get(section, option), configfilename, option)
                    else:
                        self.CheckOptions(option, parser.get(section, option), configfilename)

    # Get a list of configuration files ...
    def GetConfigFiles(self, defaults):

        if not os.path.exists(self.configdirectory):
            self.log4py.error(PIHeader)
            self.log4py.error("\n[ Main ] Error: Directory / File %s doesn't exist !\n" % self.configdirectory)
            exit(1)

        if os.path.isdir(self.configdirectory):
            if (self.configdirectory[-1:] != "/"):
                self.configdirectory = "%s/" % self.configdirectory
            files = listdir(self.configdirectory, incdirs = FALSE, inclinks = FALSE)
        else:
            files = [os.path.basename(self.configdirectory)]
            self.configdirectory = "%s/" % os.path.dirname(self.configdirectory)
            
        defaultmirror = copy.deepcopy(self.mirror)
        count = 0
        for i in range(len(files)):
            filename = files[i]
            if (filename[-1:] != "~"):
                self.mirror = copy.deepcopy(defaultmirror)
                self.ReadConfigFile(self.configdirectory + filename, defaults)
                self.mirrorlist.append(self.mirror)
                count = count + 1
        self.log4py.debug("[ Main (debug) ] %.0f configuration file(s) read" % count)

    # Signal Handler for SIGCHLD 
    def SigChldHandler(self, signal_number, stack_frame):
        # Reinstall the sigchild handler, cuz it gets deleted on libc5/sysv machines
        signal.signal(signal.SIGCHLD, self.SigChldHandler)
        global children
        pid = -1
        while (pid != 0):
            try:
                pid = os.waitpid(0, os.WNOHANG)[0]
            except:
                pid = 0
            if (pid != 0):
                if pid in children:
                    self.log4py.debug("[ Main (debug) ] Child finished")
                    children.remove(pid)

    # Delete the PID file
    def DeletePIDFile(self):
        if os.path.exists(self.pidfile):
            remove(self.pidfile)
            self.log4py.debug("[ Main (debug) ] PID File deleted")

    # Define a Handler for SigTerm for the parent
    def SigTermHandler(self, signal_number, stack_frame):
        self.log4py.info("[ Main ] main program aborted or killed - terminating children")
        global children, forkchildren
        self.log4py.debug("[ Main (debug) ] children's PIDs: %s" % children)
        forkchildren = FALSE
        for i in range(len(children)):                                          # Kill Children
            try:
                kill(children[i], signal.SIGTERM)
                kill(children[i], signal.SIGKILL)                               # Kill them again, just in case the child survived ;-)
            except:
                self.log4py.debug("[ Main (debug) ]: kill -9 %s (child) failed" % children[i])
                pass
        self.DeletePIDFile()
        exit(1)

    # Define a Handler for SigTerm for the children
    def SigTermHandlerChild(self, signal_number, stack_frame):
        exit(1)

    # Kills old EMirror Master Processes
    def KillOldProcesses(self):
        self.log4py.debug("[ Main (debug) ] reading process table & killing old mirror-processes")
        processes = process.ProcessTable()
        processes.read()
        processes.filter(filters = {"CMD": ["python", "emirror"]}, filtermode = process.FILTERMODESUB)
        ppids = processes.ppids().keys()
        for i in range(len(ppids)):
            ppid = atoi(ppids[i])
            if (ppid in processes.pids()):
                self.log4py.info("[ Main ] killing old EMirror master-process with pid %d" % ppid)
                kill(ppid, signal.SIGTERM)

    # Start mirroring ;-)
    def Start(self):
        sys.stderr = sys.stdout

        self.uid = getuid()
        self.ParseCmdLine()

        if self.configfile != None:
            self.log4py.debug("[ Main (debug) ] reading main configuration file")
            defaults = self.ReadConfigFile(self.configfile)

        if (self.configdirectory != None):
            self.log4py.debug("[ Main (debug) ] reading other configuration files")
            self.GetConfigFiles(defaults)

        if (self.maxfork == 1):
            self.log4py.info("[ Main ] ECLiPt Mirroring Tool Version %s - starting 1 copy" % PIVersion)
        else:
            self.log4py.info("[ Main ] ECLiPt Mirroring Tool Version %s - starting max. %d copies" % (PIVersion, self.maxfork))

        if (self.killprocesses == TRUE):
            self.KillOldProcesses()

        signal.signal(signal.SIGINT, self.SigTermHandler)
        signal.signal(signal.SIGTERM, self.SigTermHandler)

        signal.signal(signal.SIGCHLD, self.SigChldHandler)

        self.log4py.debug("[ Main (debug) ] writing pid-file (my pid is %d)" % getpid())
        runfile = open(self.pidfile, "w")
        runfile.write("%d" % getpid())
        runfile.close()

        global children, forkchildren
        forkchildren = TRUE
        children = []

        while len(self.mirrorlist) > 0:
            if (len(children) < self.maxfork) and (forkchildren == TRUE):
                self.log4py.debug("[ Main (debug) ] starting new child")
                mirror = self.mirrorlist[0]
                self.mirrorlist = self.mirrorlist[1:]
                self.log4py.debug("[ Main (debug) ] %d mirrors remaining" % len(self.mirrorlist))
                result = os.fork()
                if (result == 0):
                    signal.signal(signal.SIGINT, self.SigTermHandlerChild)
                    signal.signal(signal.SIGTERM, self.SigTermHandlerChild)
                    mirror.Start()
                children.append(result)
            else:
                self.log4py.debug("[ Main (debug) ] Waiting for children to finish")
                signal.pause()
        while len(children) > 0:
            self.log4py.debug("[ Main (debug) ] Waiting for children to finish")
            signal.pause()

        self.DeletePIDFile()
        
        self.log4py.debug("[ Main (debug) ] All mirrors finished")

        if (self.indexfile.directory != None):
            self.log4py.info("[ Main ] Writing Index-file in directory %s" % self.indexfile.directory)
            if self.configfile != None:
                self.ReadConfigFile(self.configfile)
            self.indexfile.Start()
