diff -r ffa851df0825 -r 2fb8b9db1c86 symbian-qemu-0.9.1-12/python-2.6.1/Tools/scripts/ftpmirror.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/symbian-qemu-0.9.1-12/python-2.6.1/Tools/scripts/ftpmirror.py Fri Jul 31 15:01:17 2009 +0100 @@ -0,0 +1,400 @@ +#! /usr/bin/env python + +"""Mirror a remote ftp subtree into a local directory tree. + +usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] + [-l username [-p passwd [-a account]]] + hostname[:port] [remotedir [localdir]] +-v: verbose +-q: quiet +-i: interactive mode +-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o') +-n: don't log in +-r: remove local files/directories no longer pertinent +-l username [-p passwd [-a account]]: login info (default .netrc or anonymous) +-s pat: skip files matching pattern +hostname: remote host w/ optional port separated by ':' +remotedir: remote directory (default initial) +localdir: local directory (default current) +""" + +import os +import sys +import time +import getopt +import ftplib +import netrc +from fnmatch import fnmatch + +# Print usage message and exit +def usage(*args): + sys.stdout = sys.stderr + for msg in args: print msg + print __doc__ + sys.exit(2) + +verbose = 1 # 0 for -q, 2 for -v +interactive = 0 +mac = 0 +rmok = 0 +nologin = 0 +skippats = ['.', '..', '.mirrorinfo'] + +# Main program: parse command line and start processing +def main(): + global verbose, interactive, mac, rmok, nologin + try: + opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') + except getopt.error, msg: + usage(msg) + login = '' + passwd = '' + account = '' + if not args: usage('hostname missing') + host = args[0] + port = 0 + if ':' in host: + host, port = host.split(':', 1) + port = int(port) + try: + auth = netrc.netrc().authenticators(host) + if auth is not None: + login, account, passwd = auth + except (netrc.NetrcParseError, IOError): + pass + for o, a in opts: + if o == '-l': login = a + if o == '-p': passwd = a + if o == '-a': account = a + if o == '-v': verbose = verbose + 1 + if o == '-q': verbose = 0 + if o == '-i': interactive = 1 + if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') + if o == '-n': nologin = 1 + if o == '-r': rmok = 1 + if o == '-s': skippats.append(a) + remotedir = '' + localdir = '' + if args[1:]: + remotedir = args[1] + if args[2:]: + localdir = args[2] + if args[3:]: usage('too many arguments') + # + f = ftplib.FTP() + if verbose: print "Connecting to '%s%s'..." % (host, + (port and ":%d"%port or "")) + f.connect(host,port) + if not nologin: + if verbose: + print 'Logging in as %r...' % (login or 'anonymous') + f.login(login, passwd, account) + if verbose: print 'OK.' + pwd = f.pwd() + if verbose > 1: print 'PWD =', repr(pwd) + if remotedir: + if verbose > 1: print 'cwd(%s)' % repr(remotedir) + f.cwd(remotedir) + if verbose > 1: print 'OK.' + pwd = f.pwd() + if verbose > 1: print 'PWD =', repr(pwd) + # + mirrorsubdir(f, localdir) + +# Core logic: mirror one subdirectory (recursively) +def mirrorsubdir(f, localdir): + pwd = f.pwd() + if localdir and not os.path.isdir(localdir): + if verbose: print 'Creating local directory', repr(localdir) + try: + makedir(localdir) + except os.error, msg: + print "Failed to establish local directory", repr(localdir) + return + infofilename = os.path.join(localdir, '.mirrorinfo') + try: + text = open(infofilename, 'r').read() + except IOError, msg: + text = '{}' + try: + info = eval(text) + except (SyntaxError, NameError): + print 'Bad mirror info in', repr(infofilename) + info = {} + subdirs = [] + listing = [] + if verbose: print 'Listing remote directory %r...' % (pwd,) + f.retrlines('LIST', listing.append) + filesfound = [] + for line in listing: + if verbose > 1: print '-->', repr(line) + if mac: + # Mac listing has just filenames; + # trailing / means subdirectory + filename = line.strip() + mode = '-' + if filename[-1:] == '/': + filename = filename[:-1] + mode = 'd' + infostuff = '' + else: + # Parse, assuming a UNIX listing + words = line.split(None, 8) + if len(words) < 6: + if verbose > 1: print 'Skipping short line' + continue + filename = words[-1].lstrip() + i = filename.find(" -> ") + if i >= 0: + # words[0] had better start with 'l'... + if verbose > 1: + print 'Found symbolic link %r' % (filename,) + linkto = filename[i+4:] + filename = filename[:i] + infostuff = words[-5:-1] + mode = words[0] + skip = 0 + for pat in skippats: + if fnmatch(filename, pat): + if verbose > 1: + print 'Skip pattern', repr(pat), + print 'matches', repr(filename) + skip = 1 + break + if skip: + continue + if mode[0] == 'd': + if verbose > 1: + print 'Remembering subdirectory', repr(filename) + subdirs.append(filename) + continue + filesfound.append(filename) + if info.has_key(filename) and info[filename] == infostuff: + if verbose > 1: + print 'Already have this version of',repr(filename) + continue + fullname = os.path.join(localdir, filename) + tempname = os.path.join(localdir, '@'+filename) + if interactive: + doit = askabout('file', filename, pwd) + if not doit: + if not info.has_key(filename): + info[filename] = 'Not retrieved' + continue + try: + os.unlink(tempname) + except os.error: + pass + if mode[0] == 'l': + if verbose: + print "Creating symlink %r -> %r" % (filename, linkto) + try: + os.symlink(linkto, tempname) + except IOError, msg: + print "Can't create %r: %s" % (tempname, msg) + continue + else: + try: + fp = open(tempname, 'wb') + except IOError, msg: + print "Can't create %r: %s" % (tempname, msg) + continue + if verbose: + print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname) + if verbose: + fp1 = LoggingFile(fp, 1024, sys.stdout) + else: + fp1 = fp + t0 = time.time() + try: + f.retrbinary('RETR ' + filename, + fp1.write, 8*1024) + except ftplib.error_perm, msg: + print msg + t1 = time.time() + bytes = fp.tell() + fp.close() + if fp1 != fp: + fp1.close() + try: + os.unlink(fullname) + except os.error: + pass # Ignore the error + try: + os.rename(tempname, fullname) + except os.error, msg: + print "Can't rename %r to %r: %s" % (tempname, fullname, msg) + continue + info[filename] = infostuff + writedict(info, infofilename) + if verbose and mode[0] != 'l': + dt = t1 - t0 + kbytes = bytes / 1024.0 + print int(round(kbytes)), + print 'Kbytes in', + print int(round(dt)), + print 'seconds', + if t1 > t0: + print '(~%d Kbytes/sec)' % \ + int(round(kbytes/dt),) + print + # + # Remove files from info that are no longer remote + deletions = 0 + for filename in info.keys(): + if filename not in filesfound: + if verbose: + print "Removing obsolete info entry for", + print repr(filename), "in", repr(localdir or ".") + del info[filename] + deletions = deletions + 1 + if deletions: + writedict(info, infofilename) + # + # Remove local files that are no longer in the remote directory + try: + if not localdir: names = os.listdir(os.curdir) + else: names = os.listdir(localdir) + except os.error: + names = [] + for name in names: + if name[0] == '.' or info.has_key(name) or name in subdirs: + continue + skip = 0 + for pat in skippats: + if fnmatch(name, pat): + if verbose > 1: + print 'Skip pattern', repr(pat), + print 'matches', repr(name) + skip = 1 + break + if skip: + continue + fullname = os.path.join(localdir, name) + if not rmok: + if verbose: + print 'Local file', repr(fullname), + print 'is no longer pertinent' + continue + if verbose: print 'Removing local file/dir', repr(fullname) + remove(fullname) + # + # Recursively mirror subdirectories + for subdir in subdirs: + if interactive: + doit = askabout('subdirectory', subdir, pwd) + if not doit: continue + if verbose: print 'Processing subdirectory', repr(subdir) + localsubdir = os.path.join(localdir, subdir) + pwd = f.pwd() + if verbose > 1: + print 'Remote directory now:', repr(pwd) + print 'Remote cwd', repr(subdir) + try: + f.cwd(subdir) + except ftplib.error_perm, msg: + print "Can't chdir to", repr(subdir), ":", repr(msg) + else: + if verbose: print 'Mirroring as', repr(localsubdir) + mirrorsubdir(f, localsubdir) + if verbose > 1: print 'Remote cwd ..' + f.cwd('..') + newpwd = f.pwd() + if newpwd != pwd: + print 'Ended up in wrong directory after cd + cd ..' + print 'Giving up now.' + break + else: + if verbose > 1: print 'OK.' + +# Helper to remove a file or directory tree +def remove(fullname): + if os.path.isdir(fullname) and not os.path.islink(fullname): + try: + names = os.listdir(fullname) + except os.error: + names = [] + ok = 1 + for name in names: + if not remove(os.path.join(fullname, name)): + ok = 0 + if not ok: + return 0 + try: + os.rmdir(fullname) + except os.error, msg: + print "Can't remove local directory %r: %s" % (fullname, msg) + return 0 + else: + try: + os.unlink(fullname) + except os.error, msg: + print "Can't remove local file %r: %s" % (fullname, msg) + return 0 + return 1 + +# Wrapper around a file for writing to write a hash sign every block. +class LoggingFile: + def __init__(self, fp, blocksize, outfp): + self.fp = fp + self.bytes = 0 + self.hashes = 0 + self.blocksize = blocksize + self.outfp = outfp + def write(self, data): + self.bytes = self.bytes + len(data) + hashes = int(self.bytes) / self.blocksize + while hashes > self.hashes: + self.outfp.write('#') + self.outfp.flush() + self.hashes = self.hashes + 1 + self.fp.write(data) + def close(self): + self.outfp.write('\n') + +# Ask permission to download a file. +def askabout(filetype, filename, pwd): + prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) + while 1: + reply = raw_input(prompt).strip().lower() + if reply in ['y', 'ye', 'yes']: + return 1 + if reply in ['', 'n', 'no', 'nop', 'nope']: + return 0 + print 'Please answer yes or no.' + +# Create a directory if it doesn't exist. Recursively create the +# parent directory as well if needed. +def makedir(pathname): + if os.path.isdir(pathname): + return + dirname = os.path.dirname(pathname) + if dirname: makedir(dirname) + os.mkdir(pathname, 0777) + +# Write a dictionary to a file in a way that can be read back using +# rval() but is still somewhat readable (i.e. not a single long line). +# Also creates a backup file. +def writedict(dict, filename): + dir, fname = os.path.split(filename) + tempname = os.path.join(dir, '@' + fname) + backup = os.path.join(dir, fname + '~') + try: + os.unlink(backup) + except os.error: + pass + fp = open(tempname, 'w') + fp.write('{\n') + for key, value in dict.items(): + fp.write('%r: %r,\n' % (key, value)) + fp.write('}\n') + fp.close() + try: + os.rename(filename, backup) + except os.error: + pass + os.rename(tempname, filename) + + +if __name__ == '__main__': + main()