|
1 #! /usr/bin/env python |
|
2 # -*- coding: iso-8859-1 -*- |
|
3 # Originally written by Barry Warsaw <barry@zope.com> |
|
4 # |
|
5 # Minimally patched to make it even more xgettext compatible |
|
6 # by Peter Funk <pf@artcom-gmbh.de> |
|
7 # |
|
8 # 2002-11-22 Jürgen Hermann <jh@web.de> |
|
9 # Added checks that _() only contains string literals, and |
|
10 # command line args are resolved to module lists, i.e. you |
|
11 # can now pass a filename, a module or package name, or a |
|
12 # directory (including globbing chars, important for Win32). |
|
13 # Made docstring fit in 80 chars wide displays using pydoc. |
|
14 # |
|
15 |
|
16 # for selftesting |
|
17 try: |
|
18 import fintl |
|
19 _ = fintl.gettext |
|
20 except ImportError: |
|
21 _ = lambda s: s |
|
22 |
|
23 __doc__ = _("""pygettext -- Python equivalent of xgettext(1) |
|
24 |
|
25 Many systems (Solaris, Linux, Gnu) provide extensive tools that ease the |
|
26 internationalization of C programs. Most of these tools are independent of |
|
27 the programming language and can be used from within Python programs. |
|
28 Martin von Loewis' work[1] helps considerably in this regard. |
|
29 |
|
30 There's one problem though; xgettext is the program that scans source code |
|
31 looking for message strings, but it groks only C (or C++). Python |
|
32 introduces a few wrinkles, such as dual quoting characters, triple quoted |
|
33 strings, and raw strings. xgettext understands none of this. |
|
34 |
|
35 Enter pygettext, which uses Python's standard tokenize module to scan |
|
36 Python source code, generating .pot files identical to what GNU xgettext[2] |
|
37 generates for C and C++ code. From there, the standard GNU tools can be |
|
38 used. |
|
39 |
|
40 A word about marking Python strings as candidates for translation. GNU |
|
41 xgettext recognizes the following keywords: gettext, dgettext, dcgettext, |
|
42 and gettext_noop. But those can be a lot of text to include all over your |
|
43 code. C and C++ have a trick: they use the C preprocessor. Most |
|
44 internationalized C source includes a #define for gettext() to _() so that |
|
45 what has to be written in the source is much less. Thus these are both |
|
46 translatable strings: |
|
47 |
|
48 gettext("Translatable String") |
|
49 _("Translatable String") |
|
50 |
|
51 Python of course has no preprocessor so this doesn't work so well. Thus, |
|
52 pygettext searches only for _() by default, but see the -k/--keyword flag |
|
53 below for how to augment this. |
|
54 |
|
55 [1] http://www.python.org/workshops/1997-10/proceedings/loewis.html |
|
56 [2] http://www.gnu.org/software/gettext/gettext.html |
|
57 |
|
58 NOTE: pygettext attempts to be option and feature compatible with GNU |
|
59 xgettext where ever possible. However some options are still missing or are |
|
60 not fully implemented. Also, xgettext's use of command line switches with |
|
61 option arguments is broken, and in these cases, pygettext just defines |
|
62 additional switches. |
|
63 |
|
64 Usage: pygettext [options] inputfile ... |
|
65 |
|
66 Options: |
|
67 |
|
68 -a |
|
69 --extract-all |
|
70 Extract all strings. |
|
71 |
|
72 -d name |
|
73 --default-domain=name |
|
74 Rename the default output file from messages.pot to name.pot. |
|
75 |
|
76 -E |
|
77 --escape |
|
78 Replace non-ASCII characters with octal escape sequences. |
|
79 |
|
80 -D |
|
81 --docstrings |
|
82 Extract module, class, method, and function docstrings. These do |
|
83 not need to be wrapped in _() markers, and in fact cannot be for |
|
84 Python to consider them docstrings. (See also the -X option). |
|
85 |
|
86 -h |
|
87 --help |
|
88 Print this help message and exit. |
|
89 |
|
90 -k word |
|
91 --keyword=word |
|
92 Keywords to look for in addition to the default set, which are: |
|
93 %(DEFAULTKEYWORDS)s |
|
94 |
|
95 You can have multiple -k flags on the command line. |
|
96 |
|
97 -K |
|
98 --no-default-keywords |
|
99 Disable the default set of keywords (see above). Any keywords |
|
100 explicitly added with the -k/--keyword option are still recognized. |
|
101 |
|
102 --no-location |
|
103 Do not write filename/lineno location comments. |
|
104 |
|
105 -n |
|
106 --add-location |
|
107 Write filename/lineno location comments indicating where each |
|
108 extracted string is found in the source. These lines appear before |
|
109 each msgid. The style of comments is controlled by the -S/--style |
|
110 option. This is the default. |
|
111 |
|
112 -o filename |
|
113 --output=filename |
|
114 Rename the default output file from messages.pot to filename. If |
|
115 filename is `-' then the output is sent to standard out. |
|
116 |
|
117 -p dir |
|
118 --output-dir=dir |
|
119 Output files will be placed in directory dir. |
|
120 |
|
121 -S stylename |
|
122 --style stylename |
|
123 Specify which style to use for location comments. Two styles are |
|
124 supported: |
|
125 |
|
126 Solaris # File: filename, line: line-number |
|
127 GNU #: filename:line |
|
128 |
|
129 The style name is case insensitive. GNU style is the default. |
|
130 |
|
131 -v |
|
132 --verbose |
|
133 Print the names of the files being processed. |
|
134 |
|
135 -V |
|
136 --version |
|
137 Print the version of pygettext and exit. |
|
138 |
|
139 -w columns |
|
140 --width=columns |
|
141 Set width of output to columns. |
|
142 |
|
143 -x filename |
|
144 --exclude-file=filename |
|
145 Specify a file that contains a list of strings that are not be |
|
146 extracted from the input files. Each string to be excluded must |
|
147 appear on a line by itself in the file. |
|
148 |
|
149 -X filename |
|
150 --no-docstrings=filename |
|
151 Specify a file that contains a list of files (one per line) that |
|
152 should not have their docstrings extracted. This is only useful in |
|
153 conjunction with the -D option above. |
|
154 |
|
155 If `inputfile' is -, standard input is read. |
|
156 """) |
|
157 |
|
158 import os |
|
159 import imp |
|
160 import sys |
|
161 import glob |
|
162 import time |
|
163 import getopt |
|
164 import token |
|
165 import tokenize |
|
166 import operator |
|
167 |
|
168 __version__ = '1.5' |
|
169 |
|
170 default_keywords = ['_'] |
|
171 DEFAULTKEYWORDS = ', '.join(default_keywords) |
|
172 |
|
173 EMPTYSTRING = '' |
|
174 |
|
175 |
|
176 |
|
177 # The normal pot-file header. msgmerge and Emacs's po-mode work better if it's |
|
178 # there. |
|
179 pot_header = _('''\ |
|
180 # SOME DESCRIPTIVE TITLE. |
|
181 # Copyright (C) YEAR ORGANIZATION |
|
182 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR. |
|
183 # |
|
184 msgid "" |
|
185 msgstr "" |
|
186 "Project-Id-Version: PACKAGE VERSION\\n" |
|
187 "POT-Creation-Date: %(time)s\\n" |
|
188 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n" |
|
189 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n" |
|
190 "Language-Team: LANGUAGE <LL@li.org>\\n" |
|
191 "MIME-Version: 1.0\\n" |
|
192 "Content-Type: text/plain; charset=CHARSET\\n" |
|
193 "Content-Transfer-Encoding: ENCODING\\n" |
|
194 "Generated-By: pygettext.py %(version)s\\n" |
|
195 |
|
196 ''') |
|
197 |
|
198 |
|
199 def usage(code, msg=''): |
|
200 print >> sys.stderr, __doc__ % globals() |
|
201 if msg: |
|
202 print >> sys.stderr, msg |
|
203 sys.exit(code) |
|
204 |
|
205 |
|
206 |
|
207 escapes = [] |
|
208 |
|
209 def make_escapes(pass_iso8859): |
|
210 global escapes |
|
211 if pass_iso8859: |
|
212 # Allow iso-8859 characters to pass through so that e.g. 'msgid |
|
213 # "Höhe"' would result not result in 'msgid "H\366he"'. Otherwise we |
|
214 # escape any character outside the 32..126 range. |
|
215 mod = 128 |
|
216 else: |
|
217 mod = 256 |
|
218 for i in range(256): |
|
219 if 32 <= (i % mod) <= 126: |
|
220 escapes.append(chr(i)) |
|
221 else: |
|
222 escapes.append("\\%03o" % i) |
|
223 escapes[ord('\\')] = '\\\\' |
|
224 escapes[ord('\t')] = '\\t' |
|
225 escapes[ord('\r')] = '\\r' |
|
226 escapes[ord('\n')] = '\\n' |
|
227 escapes[ord('\"')] = '\\"' |
|
228 |
|
229 |
|
230 def escape(s): |
|
231 global escapes |
|
232 s = list(s) |
|
233 for i in range(len(s)): |
|
234 s[i] = escapes[ord(s[i])] |
|
235 return EMPTYSTRING.join(s) |
|
236 |
|
237 |
|
238 def safe_eval(s): |
|
239 # unwrap quotes, safely |
|
240 return eval(s, {'__builtins__':{}}, {}) |
|
241 |
|
242 |
|
243 def normalize(s): |
|
244 # This converts the various Python string types into a format that is |
|
245 # appropriate for .po files, namely much closer to C style. |
|
246 lines = s.split('\n') |
|
247 if len(lines) == 1: |
|
248 s = '"' + escape(s) + '"' |
|
249 else: |
|
250 if not lines[-1]: |
|
251 del lines[-1] |
|
252 lines[-1] = lines[-1] + '\n' |
|
253 for i in range(len(lines)): |
|
254 lines[i] = escape(lines[i]) |
|
255 lineterm = '\\n"\n"' |
|
256 s = '""\n"' + lineterm.join(lines) + '"' |
|
257 return s |
|
258 |
|
259 |
|
260 def containsAny(str, set): |
|
261 """Check whether 'str' contains ANY of the chars in 'set'""" |
|
262 return 1 in [c in str for c in set] |
|
263 |
|
264 |
|
265 def _visit_pyfiles(list, dirname, names): |
|
266 """Helper for getFilesForName().""" |
|
267 # get extension for python source files |
|
268 if not globals().has_key('_py_ext'): |
|
269 global _py_ext |
|
270 _py_ext = [triple[0] for triple in imp.get_suffixes() |
|
271 if triple[2] == imp.PY_SOURCE][0] |
|
272 |
|
273 # don't recurse into CVS directories |
|
274 if 'CVS' in names: |
|
275 names.remove('CVS') |
|
276 |
|
277 # add all *.py files to list |
|
278 list.extend( |
|
279 [os.path.join(dirname, file) for file in names |
|
280 if os.path.splitext(file)[1] == _py_ext] |
|
281 ) |
|
282 |
|
283 |
|
284 def _get_modpkg_path(dotted_name, pathlist=None): |
|
285 """Get the filesystem path for a module or a package. |
|
286 |
|
287 Return the file system path to a file for a module, and to a directory for |
|
288 a package. Return None if the name is not found, or is a builtin or |
|
289 extension module. |
|
290 """ |
|
291 # split off top-most name |
|
292 parts = dotted_name.split('.', 1) |
|
293 |
|
294 if len(parts) > 1: |
|
295 # we have a dotted path, import top-level package |
|
296 try: |
|
297 file, pathname, description = imp.find_module(parts[0], pathlist) |
|
298 if file: file.close() |
|
299 except ImportError: |
|
300 return None |
|
301 |
|
302 # check if it's indeed a package |
|
303 if description[2] == imp.PKG_DIRECTORY: |
|
304 # recursively handle the remaining name parts |
|
305 pathname = _get_modpkg_path(parts[1], [pathname]) |
|
306 else: |
|
307 pathname = None |
|
308 else: |
|
309 # plain name |
|
310 try: |
|
311 file, pathname, description = imp.find_module( |
|
312 dotted_name, pathlist) |
|
313 if file: |
|
314 file.close() |
|
315 if description[2] not in [imp.PY_SOURCE, imp.PKG_DIRECTORY]: |
|
316 pathname = None |
|
317 except ImportError: |
|
318 pathname = None |
|
319 |
|
320 return pathname |
|
321 |
|
322 |
|
323 def getFilesForName(name): |
|
324 """Get a list of module files for a filename, a module or package name, |
|
325 or a directory. |
|
326 """ |
|
327 if not os.path.exists(name): |
|
328 # check for glob chars |
|
329 if containsAny(name, "*?[]"): |
|
330 files = glob.glob(name) |
|
331 list = [] |
|
332 for file in files: |
|
333 list.extend(getFilesForName(file)) |
|
334 return list |
|
335 |
|
336 # try to find module or package |
|
337 name = _get_modpkg_path(name) |
|
338 if not name: |
|
339 return [] |
|
340 |
|
341 if os.path.isdir(name): |
|
342 # find all python files in directory |
|
343 list = [] |
|
344 os.path.walk(name, _visit_pyfiles, list) |
|
345 return list |
|
346 elif os.path.exists(name): |
|
347 # a single file |
|
348 return [name] |
|
349 |
|
350 return [] |
|
351 |
|
352 |
|
353 class TokenEater: |
|
354 def __init__(self, options): |
|
355 self.__options = options |
|
356 self.__messages = {} |
|
357 self.__state = self.__waiting |
|
358 self.__data = [] |
|
359 self.__lineno = -1 |
|
360 self.__freshmodule = 1 |
|
361 self.__curfile = None |
|
362 |
|
363 def __call__(self, ttype, tstring, stup, etup, line): |
|
364 # dispatch |
|
365 ## import token |
|
366 ## print >> sys.stderr, 'ttype:', token.tok_name[ttype], \ |
|
367 ## 'tstring:', tstring |
|
368 self.__state(ttype, tstring, stup[0]) |
|
369 |
|
370 def __waiting(self, ttype, tstring, lineno): |
|
371 opts = self.__options |
|
372 # Do docstring extractions, if enabled |
|
373 if opts.docstrings and not opts.nodocstrings.get(self.__curfile): |
|
374 # module docstring? |
|
375 if self.__freshmodule: |
|
376 if ttype == tokenize.STRING: |
|
377 self.__addentry(safe_eval(tstring), lineno, isdocstring=1) |
|
378 self.__freshmodule = 0 |
|
379 elif ttype not in (tokenize.COMMENT, tokenize.NL): |
|
380 self.__freshmodule = 0 |
|
381 return |
|
382 # class docstring? |
|
383 if ttype == tokenize.NAME and tstring in ('class', 'def'): |
|
384 self.__state = self.__suiteseen |
|
385 return |
|
386 if ttype == tokenize.NAME and tstring in opts.keywords: |
|
387 self.__state = self.__keywordseen |
|
388 |
|
389 def __suiteseen(self, ttype, tstring, lineno): |
|
390 # ignore anything until we see the colon |
|
391 if ttype == tokenize.OP and tstring == ':': |
|
392 self.__state = self.__suitedocstring |
|
393 |
|
394 def __suitedocstring(self, ttype, tstring, lineno): |
|
395 # ignore any intervening noise |
|
396 if ttype == tokenize.STRING: |
|
397 self.__addentry(safe_eval(tstring), lineno, isdocstring=1) |
|
398 self.__state = self.__waiting |
|
399 elif ttype not in (tokenize.NEWLINE, tokenize.INDENT, |
|
400 tokenize.COMMENT): |
|
401 # there was no class docstring |
|
402 self.__state = self.__waiting |
|
403 |
|
404 def __keywordseen(self, ttype, tstring, lineno): |
|
405 if ttype == tokenize.OP and tstring == '(': |
|
406 self.__data = [] |
|
407 self.__lineno = lineno |
|
408 self.__state = self.__openseen |
|
409 else: |
|
410 self.__state = self.__waiting |
|
411 |
|
412 def __openseen(self, ttype, tstring, lineno): |
|
413 if ttype == tokenize.OP and tstring == ')': |
|
414 # We've seen the last of the translatable strings. Record the |
|
415 # line number of the first line of the strings and update the list |
|
416 # of messages seen. Reset state for the next batch. If there |
|
417 # were no strings inside _(), then just ignore this entry. |
|
418 if self.__data: |
|
419 self.__addentry(EMPTYSTRING.join(self.__data)) |
|
420 self.__state = self.__waiting |
|
421 elif ttype == tokenize.STRING: |
|
422 self.__data.append(safe_eval(tstring)) |
|
423 elif ttype not in [tokenize.COMMENT, token.INDENT, token.DEDENT, |
|
424 token.NEWLINE, tokenize.NL]: |
|
425 # warn if we see anything else than STRING or whitespace |
|
426 print >> sys.stderr, _( |
|
427 '*** %(file)s:%(lineno)s: Seen unexpected token "%(token)s"' |
|
428 ) % { |
|
429 'token': tstring, |
|
430 'file': self.__curfile, |
|
431 'lineno': self.__lineno |
|
432 } |
|
433 self.__state = self.__waiting |
|
434 |
|
435 def __addentry(self, msg, lineno=None, isdocstring=0): |
|
436 if lineno is None: |
|
437 lineno = self.__lineno |
|
438 if not msg in self.__options.toexclude: |
|
439 entry = (self.__curfile, lineno) |
|
440 self.__messages.setdefault(msg, {})[entry] = isdocstring |
|
441 |
|
442 def set_filename(self, filename): |
|
443 self.__curfile = filename |
|
444 self.__freshmodule = 1 |
|
445 |
|
446 def write(self, fp): |
|
447 options = self.__options |
|
448 timestamp = time.strftime('%Y-%m-%d %H:%M+%Z') |
|
449 # The time stamp in the header doesn't have the same format as that |
|
450 # generated by xgettext... |
|
451 print >> fp, pot_header % {'time': timestamp, 'version': __version__} |
|
452 # Sort the entries. First sort each particular entry's keys, then |
|
453 # sort all the entries by their first item. |
|
454 reverse = {} |
|
455 for k, v in self.__messages.items(): |
|
456 keys = v.keys() |
|
457 keys.sort() |
|
458 reverse.setdefault(tuple(keys), []).append((k, v)) |
|
459 rkeys = reverse.keys() |
|
460 rkeys.sort() |
|
461 for rkey in rkeys: |
|
462 rentries = reverse[rkey] |
|
463 rentries.sort() |
|
464 for k, v in rentries: |
|
465 isdocstring = 0 |
|
466 # If the entry was gleaned out of a docstring, then add a |
|
467 # comment stating so. This is to aid translators who may wish |
|
468 # to skip translating some unimportant docstrings. |
|
469 if reduce(operator.__add__, v.values()): |
|
470 isdocstring = 1 |
|
471 # k is the message string, v is a dictionary-set of (filename, |
|
472 # lineno) tuples. We want to sort the entries in v first by |
|
473 # file name and then by line number. |
|
474 v = v.keys() |
|
475 v.sort() |
|
476 if not options.writelocations: |
|
477 pass |
|
478 # location comments are different b/w Solaris and GNU: |
|
479 elif options.locationstyle == options.SOLARIS: |
|
480 for filename, lineno in v: |
|
481 d = {'filename': filename, 'lineno': lineno} |
|
482 print >>fp, _( |
|
483 '# File: %(filename)s, line: %(lineno)d') % d |
|
484 elif options.locationstyle == options.GNU: |
|
485 # fit as many locations on one line, as long as the |
|
486 # resulting line length doesn't exceeds 'options.width' |
|
487 locline = '#:' |
|
488 for filename, lineno in v: |
|
489 d = {'filename': filename, 'lineno': lineno} |
|
490 s = _(' %(filename)s:%(lineno)d') % d |
|
491 if len(locline) + len(s) <= options.width: |
|
492 locline = locline + s |
|
493 else: |
|
494 print >> fp, locline |
|
495 locline = "#:" + s |
|
496 if len(locline) > 2: |
|
497 print >> fp, locline |
|
498 if isdocstring: |
|
499 print >> fp, '#, docstring' |
|
500 print >> fp, 'msgid', normalize(k) |
|
501 print >> fp, 'msgstr ""\n' |
|
502 |
|
503 |
|
504 |
|
505 def main(): |
|
506 global default_keywords |
|
507 try: |
|
508 opts, args = getopt.getopt( |
|
509 sys.argv[1:], |
|
510 'ad:DEhk:Kno:p:S:Vvw:x:X:', |
|
511 ['extract-all', 'default-domain=', 'escape', 'help', |
|
512 'keyword=', 'no-default-keywords', |
|
513 'add-location', 'no-location', 'output=', 'output-dir=', |
|
514 'style=', 'verbose', 'version', 'width=', 'exclude-file=', |
|
515 'docstrings', 'no-docstrings', |
|
516 ]) |
|
517 except getopt.error, msg: |
|
518 usage(1, msg) |
|
519 |
|
520 # for holding option values |
|
521 class Options: |
|
522 # constants |
|
523 GNU = 1 |
|
524 SOLARIS = 2 |
|
525 # defaults |
|
526 extractall = 0 # FIXME: currently this option has no effect at all. |
|
527 escape = 0 |
|
528 keywords = [] |
|
529 outpath = '' |
|
530 outfile = 'messages.pot' |
|
531 writelocations = 1 |
|
532 locationstyle = GNU |
|
533 verbose = 0 |
|
534 width = 78 |
|
535 excludefilename = '' |
|
536 docstrings = 0 |
|
537 nodocstrings = {} |
|
538 |
|
539 options = Options() |
|
540 locations = {'gnu' : options.GNU, |
|
541 'solaris' : options.SOLARIS, |
|
542 } |
|
543 |
|
544 # parse options |
|
545 for opt, arg in opts: |
|
546 if opt in ('-h', '--help'): |
|
547 usage(0) |
|
548 elif opt in ('-a', '--extract-all'): |
|
549 options.extractall = 1 |
|
550 elif opt in ('-d', '--default-domain'): |
|
551 options.outfile = arg + '.pot' |
|
552 elif opt in ('-E', '--escape'): |
|
553 options.escape = 1 |
|
554 elif opt in ('-D', '--docstrings'): |
|
555 options.docstrings = 1 |
|
556 elif opt in ('-k', '--keyword'): |
|
557 options.keywords.append(arg) |
|
558 elif opt in ('-K', '--no-default-keywords'): |
|
559 default_keywords = [] |
|
560 elif opt in ('-n', '--add-location'): |
|
561 options.writelocations = 1 |
|
562 elif opt in ('--no-location',): |
|
563 options.writelocations = 0 |
|
564 elif opt in ('-S', '--style'): |
|
565 options.locationstyle = locations.get(arg.lower()) |
|
566 if options.locationstyle is None: |
|
567 usage(1, _('Invalid value for --style: %s') % arg) |
|
568 elif opt in ('-o', '--output'): |
|
569 options.outfile = arg |
|
570 elif opt in ('-p', '--output-dir'): |
|
571 options.outpath = arg |
|
572 elif opt in ('-v', '--verbose'): |
|
573 options.verbose = 1 |
|
574 elif opt in ('-V', '--version'): |
|
575 print _('pygettext.py (xgettext for Python) %s') % __version__ |
|
576 sys.exit(0) |
|
577 elif opt in ('-w', '--width'): |
|
578 try: |
|
579 options.width = int(arg) |
|
580 except ValueError: |
|
581 usage(1, _('--width argument must be an integer: %s') % arg) |
|
582 elif opt in ('-x', '--exclude-file'): |
|
583 options.excludefilename = arg |
|
584 elif opt in ('-X', '--no-docstrings'): |
|
585 fp = open(arg) |
|
586 try: |
|
587 while 1: |
|
588 line = fp.readline() |
|
589 if not line: |
|
590 break |
|
591 options.nodocstrings[line[:-1]] = 1 |
|
592 finally: |
|
593 fp.close() |
|
594 |
|
595 # calculate escapes |
|
596 make_escapes(options.escape) |
|
597 |
|
598 # calculate all keywords |
|
599 options.keywords.extend(default_keywords) |
|
600 |
|
601 # initialize list of strings to exclude |
|
602 if options.excludefilename: |
|
603 try: |
|
604 fp = open(options.excludefilename) |
|
605 options.toexclude = fp.readlines() |
|
606 fp.close() |
|
607 except IOError: |
|
608 print >> sys.stderr, _( |
|
609 "Can't read --exclude-file: %s") % options.excludefilename |
|
610 sys.exit(1) |
|
611 else: |
|
612 options.toexclude = [] |
|
613 |
|
614 # resolve args to module lists |
|
615 expanded = [] |
|
616 for arg in args: |
|
617 if arg == '-': |
|
618 expanded.append(arg) |
|
619 else: |
|
620 expanded.extend(getFilesForName(arg)) |
|
621 args = expanded |
|
622 |
|
623 # slurp through all the files |
|
624 eater = TokenEater(options) |
|
625 for filename in args: |
|
626 if filename == '-': |
|
627 if options.verbose: |
|
628 print _('Reading standard input') |
|
629 fp = sys.stdin |
|
630 closep = 0 |
|
631 else: |
|
632 if options.verbose: |
|
633 print _('Working on %s') % filename |
|
634 fp = open(filename) |
|
635 closep = 1 |
|
636 try: |
|
637 eater.set_filename(filename) |
|
638 try: |
|
639 tokenize.tokenize(fp.readline, eater) |
|
640 except tokenize.TokenError, e: |
|
641 print >> sys.stderr, '%s: %s, line %d, column %d' % ( |
|
642 e[0], filename, e[1][0], e[1][1]) |
|
643 finally: |
|
644 if closep: |
|
645 fp.close() |
|
646 |
|
647 # write the output |
|
648 if options.outfile == '-': |
|
649 fp = sys.stdout |
|
650 closep = 0 |
|
651 else: |
|
652 if options.outpath: |
|
653 options.outfile = os.path.join(options.outpath, options.outfile) |
|
654 fp = open(options.outfile, 'w') |
|
655 closep = 1 |
|
656 try: |
|
657 eater.write(fp) |
|
658 finally: |
|
659 if closep: |
|
660 fp.close() |
|
661 |
|
662 |
|
663 if __name__ == '__main__': |
|
664 main() |
|
665 # some more test strings |
|
666 _(u'a unicode string') |
|
667 # this one creates a warning |
|
668 _('*** Seen unexpected token "%(token)s"') % {'token': 'test'} |
|
669 _('more' 'than' 'one' 'string') |