|
1 #! /usr/bin/env python |
|
2 |
|
3 """Mirror a remote ftp subtree into a local directory tree. |
|
4 |
|
5 usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] |
|
6 [-l username [-p passwd [-a account]]] |
|
7 hostname[:port] [remotedir [localdir]] |
|
8 -v: verbose |
|
9 -q: quiet |
|
10 -i: interactive mode |
|
11 -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o') |
|
12 -n: don't log in |
|
13 -r: remove local files/directories no longer pertinent |
|
14 -l username [-p passwd [-a account]]: login info (default .netrc or anonymous) |
|
15 -s pat: skip files matching pattern |
|
16 hostname: remote host w/ optional port separated by ':' |
|
17 remotedir: remote directory (default initial) |
|
18 localdir: local directory (default current) |
|
19 """ |
|
20 |
|
21 import os |
|
22 import sys |
|
23 import time |
|
24 import getopt |
|
25 import ftplib |
|
26 import netrc |
|
27 from fnmatch import fnmatch |
|
28 |
|
29 # Print usage message and exit |
|
30 def usage(*args): |
|
31 sys.stdout = sys.stderr |
|
32 for msg in args: print msg |
|
33 print __doc__ |
|
34 sys.exit(2) |
|
35 |
|
36 verbose = 1 # 0 for -q, 2 for -v |
|
37 interactive = 0 |
|
38 mac = 0 |
|
39 rmok = 0 |
|
40 nologin = 0 |
|
41 skippats = ['.', '..', '.mirrorinfo'] |
|
42 |
|
43 # Main program: parse command line and start processing |
|
44 def main(): |
|
45 global verbose, interactive, mac, rmok, nologin |
|
46 try: |
|
47 opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') |
|
48 except getopt.error, msg: |
|
49 usage(msg) |
|
50 login = '' |
|
51 passwd = '' |
|
52 account = '' |
|
53 if not args: usage('hostname missing') |
|
54 host = args[0] |
|
55 port = 0 |
|
56 if ':' in host: |
|
57 host, port = host.split(':', 1) |
|
58 port = int(port) |
|
59 try: |
|
60 auth = netrc.netrc().authenticators(host) |
|
61 if auth is not None: |
|
62 login, account, passwd = auth |
|
63 except (netrc.NetrcParseError, IOError): |
|
64 pass |
|
65 for o, a in opts: |
|
66 if o == '-l': login = a |
|
67 if o == '-p': passwd = a |
|
68 if o == '-a': account = a |
|
69 if o == '-v': verbose = verbose + 1 |
|
70 if o == '-q': verbose = 0 |
|
71 if o == '-i': interactive = 1 |
|
72 if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') |
|
73 if o == '-n': nologin = 1 |
|
74 if o == '-r': rmok = 1 |
|
75 if o == '-s': skippats.append(a) |
|
76 remotedir = '' |
|
77 localdir = '' |
|
78 if args[1:]: |
|
79 remotedir = args[1] |
|
80 if args[2:]: |
|
81 localdir = args[2] |
|
82 if args[3:]: usage('too many arguments') |
|
83 # |
|
84 f = ftplib.FTP() |
|
85 if verbose: print "Connecting to '%s%s'..." % (host, |
|
86 (port and ":%d"%port or "")) |
|
87 f.connect(host,port) |
|
88 if not nologin: |
|
89 if verbose: |
|
90 print 'Logging in as %r...' % (login or 'anonymous') |
|
91 f.login(login, passwd, account) |
|
92 if verbose: print 'OK.' |
|
93 pwd = f.pwd() |
|
94 if verbose > 1: print 'PWD =', repr(pwd) |
|
95 if remotedir: |
|
96 if verbose > 1: print 'cwd(%s)' % repr(remotedir) |
|
97 f.cwd(remotedir) |
|
98 if verbose > 1: print 'OK.' |
|
99 pwd = f.pwd() |
|
100 if verbose > 1: print 'PWD =', repr(pwd) |
|
101 # |
|
102 mirrorsubdir(f, localdir) |
|
103 |
|
104 # Core logic: mirror one subdirectory (recursively) |
|
105 def mirrorsubdir(f, localdir): |
|
106 pwd = f.pwd() |
|
107 if localdir and not os.path.isdir(localdir): |
|
108 if verbose: print 'Creating local directory', repr(localdir) |
|
109 try: |
|
110 makedir(localdir) |
|
111 except os.error, msg: |
|
112 print "Failed to establish local directory", repr(localdir) |
|
113 return |
|
114 infofilename = os.path.join(localdir, '.mirrorinfo') |
|
115 try: |
|
116 text = open(infofilename, 'r').read() |
|
117 except IOError, msg: |
|
118 text = '{}' |
|
119 try: |
|
120 info = eval(text) |
|
121 except (SyntaxError, NameError): |
|
122 print 'Bad mirror info in', repr(infofilename) |
|
123 info = {} |
|
124 subdirs = [] |
|
125 listing = [] |
|
126 if verbose: print 'Listing remote directory %r...' % (pwd,) |
|
127 f.retrlines('LIST', listing.append) |
|
128 filesfound = [] |
|
129 for line in listing: |
|
130 if verbose > 1: print '-->', repr(line) |
|
131 if mac: |
|
132 # Mac listing has just filenames; |
|
133 # trailing / means subdirectory |
|
134 filename = line.strip() |
|
135 mode = '-' |
|
136 if filename[-1:] == '/': |
|
137 filename = filename[:-1] |
|
138 mode = 'd' |
|
139 infostuff = '' |
|
140 else: |
|
141 # Parse, assuming a UNIX listing |
|
142 words = line.split(None, 8) |
|
143 if len(words) < 6: |
|
144 if verbose > 1: print 'Skipping short line' |
|
145 continue |
|
146 filename = words[-1].lstrip() |
|
147 i = filename.find(" -> ") |
|
148 if i >= 0: |
|
149 # words[0] had better start with 'l'... |
|
150 if verbose > 1: |
|
151 print 'Found symbolic link %r' % (filename,) |
|
152 linkto = filename[i+4:] |
|
153 filename = filename[:i] |
|
154 infostuff = words[-5:-1] |
|
155 mode = words[0] |
|
156 skip = 0 |
|
157 for pat in skippats: |
|
158 if fnmatch(filename, pat): |
|
159 if verbose > 1: |
|
160 print 'Skip pattern', repr(pat), |
|
161 print 'matches', repr(filename) |
|
162 skip = 1 |
|
163 break |
|
164 if skip: |
|
165 continue |
|
166 if mode[0] == 'd': |
|
167 if verbose > 1: |
|
168 print 'Remembering subdirectory', repr(filename) |
|
169 subdirs.append(filename) |
|
170 continue |
|
171 filesfound.append(filename) |
|
172 if info.has_key(filename) and info[filename] == infostuff: |
|
173 if verbose > 1: |
|
174 print 'Already have this version of',repr(filename) |
|
175 continue |
|
176 fullname = os.path.join(localdir, filename) |
|
177 tempname = os.path.join(localdir, '@'+filename) |
|
178 if interactive: |
|
179 doit = askabout('file', filename, pwd) |
|
180 if not doit: |
|
181 if not info.has_key(filename): |
|
182 info[filename] = 'Not retrieved' |
|
183 continue |
|
184 try: |
|
185 os.unlink(tempname) |
|
186 except os.error: |
|
187 pass |
|
188 if mode[0] == 'l': |
|
189 if verbose: |
|
190 print "Creating symlink %r -> %r" % (filename, linkto) |
|
191 try: |
|
192 os.symlink(linkto, tempname) |
|
193 except IOError, msg: |
|
194 print "Can't create %r: %s" % (tempname, msg) |
|
195 continue |
|
196 else: |
|
197 try: |
|
198 fp = open(tempname, 'wb') |
|
199 except IOError, msg: |
|
200 print "Can't create %r: %s" % (tempname, msg) |
|
201 continue |
|
202 if verbose: |
|
203 print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname) |
|
204 if verbose: |
|
205 fp1 = LoggingFile(fp, 1024, sys.stdout) |
|
206 else: |
|
207 fp1 = fp |
|
208 t0 = time.time() |
|
209 try: |
|
210 f.retrbinary('RETR ' + filename, |
|
211 fp1.write, 8*1024) |
|
212 except ftplib.error_perm, msg: |
|
213 print msg |
|
214 t1 = time.time() |
|
215 bytes = fp.tell() |
|
216 fp.close() |
|
217 if fp1 != fp: |
|
218 fp1.close() |
|
219 try: |
|
220 os.unlink(fullname) |
|
221 except os.error: |
|
222 pass # Ignore the error |
|
223 try: |
|
224 os.rename(tempname, fullname) |
|
225 except os.error, msg: |
|
226 print "Can't rename %r to %r: %s" % (tempname, fullname, msg) |
|
227 continue |
|
228 info[filename] = infostuff |
|
229 writedict(info, infofilename) |
|
230 if verbose and mode[0] != 'l': |
|
231 dt = t1 - t0 |
|
232 kbytes = bytes / 1024.0 |
|
233 print int(round(kbytes)), |
|
234 print 'Kbytes in', |
|
235 print int(round(dt)), |
|
236 print 'seconds', |
|
237 if t1 > t0: |
|
238 print '(~%d Kbytes/sec)' % \ |
|
239 int(round(kbytes/dt),) |
|
240 print |
|
241 # |
|
242 # Remove files from info that are no longer remote |
|
243 deletions = 0 |
|
244 for filename in info.keys(): |
|
245 if filename not in filesfound: |
|
246 if verbose: |
|
247 print "Removing obsolete info entry for", |
|
248 print repr(filename), "in", repr(localdir or ".") |
|
249 del info[filename] |
|
250 deletions = deletions + 1 |
|
251 if deletions: |
|
252 writedict(info, infofilename) |
|
253 # |
|
254 # Remove local files that are no longer in the remote directory |
|
255 try: |
|
256 if not localdir: names = os.listdir(os.curdir) |
|
257 else: names = os.listdir(localdir) |
|
258 except os.error: |
|
259 names = [] |
|
260 for name in names: |
|
261 if name[0] == '.' or info.has_key(name) or name in subdirs: |
|
262 continue |
|
263 skip = 0 |
|
264 for pat in skippats: |
|
265 if fnmatch(name, pat): |
|
266 if verbose > 1: |
|
267 print 'Skip pattern', repr(pat), |
|
268 print 'matches', repr(name) |
|
269 skip = 1 |
|
270 break |
|
271 if skip: |
|
272 continue |
|
273 fullname = os.path.join(localdir, name) |
|
274 if not rmok: |
|
275 if verbose: |
|
276 print 'Local file', repr(fullname), |
|
277 print 'is no longer pertinent' |
|
278 continue |
|
279 if verbose: print 'Removing local file/dir', repr(fullname) |
|
280 remove(fullname) |
|
281 # |
|
282 # Recursively mirror subdirectories |
|
283 for subdir in subdirs: |
|
284 if interactive: |
|
285 doit = askabout('subdirectory', subdir, pwd) |
|
286 if not doit: continue |
|
287 if verbose: print 'Processing subdirectory', repr(subdir) |
|
288 localsubdir = os.path.join(localdir, subdir) |
|
289 pwd = f.pwd() |
|
290 if verbose > 1: |
|
291 print 'Remote directory now:', repr(pwd) |
|
292 print 'Remote cwd', repr(subdir) |
|
293 try: |
|
294 f.cwd(subdir) |
|
295 except ftplib.error_perm, msg: |
|
296 print "Can't chdir to", repr(subdir), ":", repr(msg) |
|
297 else: |
|
298 if verbose: print 'Mirroring as', repr(localsubdir) |
|
299 mirrorsubdir(f, localsubdir) |
|
300 if verbose > 1: print 'Remote cwd ..' |
|
301 f.cwd('..') |
|
302 newpwd = f.pwd() |
|
303 if newpwd != pwd: |
|
304 print 'Ended up in wrong directory after cd + cd ..' |
|
305 print 'Giving up now.' |
|
306 break |
|
307 else: |
|
308 if verbose > 1: print 'OK.' |
|
309 |
|
310 # Helper to remove a file or directory tree |
|
311 def remove(fullname): |
|
312 if os.path.isdir(fullname) and not os.path.islink(fullname): |
|
313 try: |
|
314 names = os.listdir(fullname) |
|
315 except os.error: |
|
316 names = [] |
|
317 ok = 1 |
|
318 for name in names: |
|
319 if not remove(os.path.join(fullname, name)): |
|
320 ok = 0 |
|
321 if not ok: |
|
322 return 0 |
|
323 try: |
|
324 os.rmdir(fullname) |
|
325 except os.error, msg: |
|
326 print "Can't remove local directory %r: %s" % (fullname, msg) |
|
327 return 0 |
|
328 else: |
|
329 try: |
|
330 os.unlink(fullname) |
|
331 except os.error, msg: |
|
332 print "Can't remove local file %r: %s" % (fullname, msg) |
|
333 return 0 |
|
334 return 1 |
|
335 |
|
336 # Wrapper around a file for writing to write a hash sign every block. |
|
337 class LoggingFile: |
|
338 def __init__(self, fp, blocksize, outfp): |
|
339 self.fp = fp |
|
340 self.bytes = 0 |
|
341 self.hashes = 0 |
|
342 self.blocksize = blocksize |
|
343 self.outfp = outfp |
|
344 def write(self, data): |
|
345 self.bytes = self.bytes + len(data) |
|
346 hashes = int(self.bytes) / self.blocksize |
|
347 while hashes > self.hashes: |
|
348 self.outfp.write('#') |
|
349 self.outfp.flush() |
|
350 self.hashes = self.hashes + 1 |
|
351 self.fp.write(data) |
|
352 def close(self): |
|
353 self.outfp.write('\n') |
|
354 |
|
355 # Ask permission to download a file. |
|
356 def askabout(filetype, filename, pwd): |
|
357 prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) |
|
358 while 1: |
|
359 reply = raw_input(prompt).strip().lower() |
|
360 if reply in ['y', 'ye', 'yes']: |
|
361 return 1 |
|
362 if reply in ['', 'n', 'no', 'nop', 'nope']: |
|
363 return 0 |
|
364 print 'Please answer yes or no.' |
|
365 |
|
366 # Create a directory if it doesn't exist. Recursively create the |
|
367 # parent directory as well if needed. |
|
368 def makedir(pathname): |
|
369 if os.path.isdir(pathname): |
|
370 return |
|
371 dirname = os.path.dirname(pathname) |
|
372 if dirname: makedir(dirname) |
|
373 os.mkdir(pathname, 0777) |
|
374 |
|
375 # Write a dictionary to a file in a way that can be read back using |
|
376 # rval() but is still somewhat readable (i.e. not a single long line). |
|
377 # Also creates a backup file. |
|
378 def writedict(dict, filename): |
|
379 dir, fname = os.path.split(filename) |
|
380 tempname = os.path.join(dir, '@' + fname) |
|
381 backup = os.path.join(dir, fname + '~') |
|
382 try: |
|
383 os.unlink(backup) |
|
384 except os.error: |
|
385 pass |
|
386 fp = open(tempname, 'w') |
|
387 fp.write('{\n') |
|
388 for key, value in dict.items(): |
|
389 fp.write('%r: %r,\n' % (key, value)) |
|
390 fp.write('}\n') |
|
391 fp.close() |
|
392 try: |
|
393 os.rename(filename, backup) |
|
394 except os.error: |
|
395 pass |
|
396 os.rename(tempname, filename) |
|
397 |
|
398 |
|
399 if __name__ == '__main__': |
|
400 main() |