#! /usr/bin/env python

"""Mirror a remote ftp subtree into a local directory tree.

usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]

                 [-l username [-p passwd [-a account]]]

                 hostname[:port] [remotedir [localdir]]

-v: verbose

-q: quiet

-i: interactive mode

-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')

-n: don't log in

-r: remove local files/directories no longer pertinent

-l username [-p passwd [-a account]]: login info (default .netrc or anonymous)

-s pat: skip files matching pattern

hostname: remote host w/ optional port separated by ':'

remotedir: remote directory (default initial)

localdir: local directory (default current)

"""

import os

import sys

import time

import getopt

import ftplib

import netrc

from fnmatch import fnmatch

# Print usage message and exit

def usage(*args):

    sys.stdout = sys.stderr

    for msg in args: print msg

    print __doc__

    sys.exit(2)

verbose = 1 # 0 for -q, 2 for -v

interactive = 0

mac = 0

rmok = 0

nologin = 0

skippats = ['.', '..', '.mirrorinfo']

# Main program: parse command line and start processing

def main():

    global verbose, interactive, mac, rmok, nologin

    try:

        opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')

    except getopt.error, msg:

        usage(msg)

    login = ''

    passwd = ''

    account = ''

    if not args: usage('hostname missing')

    host = args[0]

    port = 0

    if ':' in host:

        host, port = host.split(':', 1)

        port = int(port)

    try:

        auth = netrc.netrc().authenticators(host)

        if auth is not None:

            login, account, passwd = auth

    except (netrc.NetrcParseError, IOError):

        pass

    for o, a in opts:

        if o == '-l': login = a

        if o == '-p': passwd = a

        if o == '-a': account = a

        if o == '-v': verbose = verbose + 1

        if o == '-q': verbose = 0

        if o == '-i': interactive = 1

        if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')

        if o == '-n': nologin = 1

        if o == '-r': rmok = 1

        if o == '-s': skippats.append(a)

    remotedir = ''

    localdir = ''

    if args[1:]:

        remotedir = args[1]

        if args[2:]:

            localdir = args[2]

            if args[3:]: usage('too many arguments')

    #

    f = ftplib.FTP()

    if verbose: print "Connecting to '%s%s'..." % (host,

                                                   (port and ":%d"%port or ""))

    f.connect(host,port)

    if not nologin:

        if verbose:

            print 'Logging in as %r...' % (login or 'anonymous')

        f.login(login, passwd, account)

    if verbose: print 'OK.'

    pwd = f.pwd()

    if verbose > 1: print 'PWD =', repr(pwd)

    if remotedir:

        if verbose > 1: print 'cwd(%s)' % repr(remotedir)

        f.cwd(remotedir)

        if verbose > 1: print 'OK.'

        pwd = f.pwd()

        if verbose > 1: print 'PWD =', repr(pwd)

    #

    mirrorsubdir(f, localdir)

# Core logic: mirror one subdirectory (recursively)

def mirrorsubdir(f, localdir):

    pwd = f.pwd()

    if localdir and not os.path.isdir(localdir):

        if verbose: print 'Creating local directory', repr(localdir)

        try:

            makedir(localdir)

        except os.error, msg:

            print "Failed to establish local directory", repr(localdir)

            return

    infofilename = os.path.join(localdir, '.mirrorinfo')

    try:

        text = open(infofilename, 'r').read()

    except IOError, msg:

        text = '{}'

    try:

        info = eval(text)

    except (SyntaxError, NameError):

        print 'Bad mirror info in', repr(infofilename)

        info = {}

    subdirs = []

    listing = []

    if verbose: print 'Listing remote directory %r...' % (pwd,)

    f.retrlines('LIST', listing.append)

    filesfound = []

    for line in listing:

        if verbose > 1: print '-->', repr(line)

        if mac:

            # Mac listing has just filenames;

            # trailing / means subdirectory

            filename = line.strip()

            mode = '-'

            if filename[-1:] == '/':

                filename = filename[:-1]

                mode = 'd'

            infostuff = ''

        else:

            # Parse, assuming a UNIX listing

            words = line.split(None, 8)

            if len(words) < 6:

                if verbose > 1: print 'Skipping short line'

                continue

            filename = words[-1].lstrip()

            i = filename.find(" -> ")

            if i >= 0:

                # words[0] had better start with 'l'...

                if verbose > 1:

                    print 'Found symbolic link %r' % (filename,)

                linkto = filename[i+4:]

                filename = filename[:i]

            infostuff = words[-5:-1]

            mode = words[0]

        skip = 0

        for pat in skippats:

            if fnmatch(filename, pat):

                if verbose > 1:

                    print 'Skip pattern', repr(pat),

                    print 'matches', repr(filename)

                skip = 1

                break

        if skip:

            continue

        if mode[0] == 'd':

            if verbose > 1:

                print 'Remembering subdirectory', repr(filename)

            subdirs.append(filename)

            continue

        filesfound.append(filename)

        if info.has_key(filename) and info[filename] == infostuff:

            if verbose > 1:

                print 'Already have this version of',repr(filename)

            continue

        fullname = os.path.join(localdir, filename)

        tempname = os.path.join(localdir, '@'+filename)

        if interactive:

            doit = askabout('file', filename, pwd)

            if not doit:

                if not info.has_key(filename):

                    info[filename] = 'Not retrieved'

                continue

        try:

            os.unlink(tempname)

        except os.error:

            pass

        if mode[0] == 'l':

            if verbose:

                print "Creating symlink %r -> %r" % (filename, linkto)

            try:

                os.symlink(linkto, tempname)

            except IOError, msg:

                print "Can't create %r: %s" % (tempname, msg)

                continue

        else:

            try:

                fp = open(tempname, 'wb')

            except IOError, msg:

                print "Can't create %r: %s" % (tempname, msg)

                continue

            if verbose:

                print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname)

            if verbose:

                fp1 = LoggingFile(fp, 1024, sys.stdout)

            else:

                fp1 = fp

            t0 = time.time()

            try:

                f.retrbinary('RETR ' + filename,

                             fp1.write, 8*1024)

            except ftplib.error_perm, msg:

                print msg

            t1 = time.time()

            bytes = fp.tell()

            fp.close()

            if fp1 != fp:

                fp1.close()

        try:

            os.unlink(fullname)

        except os.error:

            pass            # Ignore the error

        try:

            os.rename(tempname, fullname)

        except os.error, msg:

            print "Can't rename %r to %r: %s" % (tempname, fullname, msg)

            continue

        info[filename] = infostuff

        writedict(info, infofilename)

        if verbose and mode[0] != 'l':

            dt = t1 - t0

            kbytes = bytes / 1024.0

            print int(round(kbytes)),

            print 'Kbytes in',

            print int(round(dt)),

            print 'seconds',

            if t1 > t0:

                print '(~%d Kbytes/sec)' % \

                          int(round(kbytes/dt),)

            print

    #

    # Remove files from info that are no longer remote

    deletions = 0

    for filename in info.keys():

        if filename not in filesfound:

            if verbose:

                print "Removing obsolete info entry for",

                print repr(filename), "in", repr(localdir or ".")

            del info[filename]

            deletions = deletions + 1

    if deletions:

        writedict(info, infofilename)

    #

    # Remove local files that are no longer in the remote directory

    try:

        if not localdir: names = os.listdir(os.curdir)

        else: names = os.listdir(localdir)

    except os.error:

        names = []

    for name in names:

        if name[0] == '.' or info.has_key(name) or name in subdirs:

            continue

        skip = 0

        for pat in skippats:

            if fnmatch(name, pat):

                if verbose > 1:

                    print 'Skip pattern', repr(pat),

                    print 'matches', repr(name)

                skip = 1

                break

        if skip:

            continue

        fullname = os.path.join(localdir, name)

        if not rmok:

            if verbose:

                print 'Local file', repr(fullname),

                print 'is no longer pertinent'

            continue

        if verbose: print 'Removing local file/dir', repr(fullname)

        remove(fullname)

    #

    # Recursively mirror subdirectories

    for subdir in subdirs:

        if interactive:

            doit = askabout('subdirectory', subdir, pwd)

            if not doit: continue

        if verbose: print 'Processing subdirectory', repr(subdir)

        localsubdir = os.path.join(localdir, subdir)

        pwd = f.pwd()

        if verbose > 1:

            print 'Remote directory now:', repr(pwd)

            print 'Remote cwd', repr(subdir)

        try:

            f.cwd(subdir)

        except ftplib.error_perm, msg:

            print "Can't chdir to", repr(subdir), ":", repr(msg)

        else:

            if verbose: print 'Mirroring as', repr(localsubdir)

            mirrorsubdir(f, localsubdir)

            if verbose > 1: print 'Remote cwd ..'

            f.cwd('..')

        newpwd = f.pwd()

        if newpwd != pwd:

            print 'Ended up in wrong directory after cd + cd ..'

            print 'Giving up now.'

            break

        else:

            if verbose > 1: print 'OK.'

# Helper to remove a file or directory tree

def remove(fullname):

    if os.path.isdir(fullname) and not os.path.islink(fullname):

        try:

            names = os.listdir(fullname)

        except os.error:

            names = []

        ok = 1

        for name in names:

            if not remove(os.path.join(fullname, name)):

                ok = 0

        if not ok:

            return 0

        try:

            os.rmdir(fullname)

        except os.error, msg:

            print "Can't remove local directory %r: %s" % (fullname, msg)

            return 0

    else:

        try:

            os.unlink(fullname)

        except os.error, msg:

            print "Can't remove local file %r: %s" % (fullname, msg)

            return 0

    return 1

# Wrapper around a file for writing to write a hash sign every block.

class LoggingFile:

    def __init__(self, fp, blocksize, outfp):

        self.fp = fp

        self.bytes = 0

        self.hashes = 0

        self.blocksize = blocksize

        self.outfp = outfp

    def write(self, data):

        self.bytes = self.bytes + len(data)

        hashes = int(self.bytes) / self.blocksize

        while hashes > self.hashes:

            self.outfp.write('#')

            self.outfp.flush()

            self.hashes = self.hashes + 1

        self.fp.write(data)

    def close(self):

        self.outfp.write('\n')

# Ask permission to download a file.

def askabout(filetype, filename, pwd):

    prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)

    while 1:

        reply = raw_input(prompt).strip().lower()

        if reply in ['y', 'ye', 'yes']:

            return 1

        if reply in ['', 'n', 'no', 'nop', 'nope']:

            return 0

        print 'Please answer yes or no.'

# Create a directory if it doesn't exist.  Recursively create the

# parent directory as well if needed.

def makedir(pathname):

    if os.path.isdir(pathname):

        return

    dirname = os.path.dirname(pathname)

    if dirname: makedir(dirname)

    os.mkdir(pathname, 0777)

# Write a dictionary to a file in a way that can be read back using

# rval() but is still somewhat readable (i.e. not a single long line).

# Also creates a backup file.

def writedict(dict, filename):

    dir, fname = os.path.split(filename)

    tempname = os.path.join(dir, '@' + fname)

    backup = os.path.join(dir, fname + '~')

    try:

        os.unlink(backup)

    except os.error:

        pass

    fp = open(tempname, 'w')

    fp.write('{\n')

    for key, value in dict.items():

        fp.write('%r: %r,\n' % (key, value))

    fp.write('}\n')

    fp.close()

    try:

        os.rename(filename, backup)

    except os.error:

        pass

    os.rename(tempname, filename)

if __name__ == '__main__':

    main()