|
1 """Utilities to support packages.""" |
|
2 |
|
3 # NOTE: This module must remain compatible with Python 2.3, as it is shared |
|
4 # by setuptools for distribution with Python 2.3 and up. |
|
5 |
|
6 import os |
|
7 import sys |
|
8 import imp |
|
9 import os.path |
|
10 from types import ModuleType |
|
11 |
|
12 __all__ = [ |
|
13 'get_importer', 'iter_importers', 'get_loader', 'find_loader', |
|
14 'walk_packages', 'iter_modules', |
|
15 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', |
|
16 ] |
|
17 |
|
18 def read_code(stream): |
|
19 # This helper is needed in order for the PEP 302 emulation to |
|
20 # correctly handle compiled files |
|
21 import marshal |
|
22 |
|
23 magic = stream.read(4) |
|
24 if magic != imp.get_magic(): |
|
25 return None |
|
26 |
|
27 stream.read(4) # Skip timestamp |
|
28 return marshal.load(stream) |
|
29 |
|
30 |
|
31 def simplegeneric(func): |
|
32 """Make a trivial single-dispatch generic function""" |
|
33 registry = {} |
|
34 def wrapper(*args, **kw): |
|
35 ob = args[0] |
|
36 try: |
|
37 cls = ob.__class__ |
|
38 except AttributeError: |
|
39 cls = type(ob) |
|
40 try: |
|
41 mro = cls.__mro__ |
|
42 except AttributeError: |
|
43 try: |
|
44 class cls(cls, object): |
|
45 pass |
|
46 mro = cls.__mro__[1:] |
|
47 except TypeError: |
|
48 mro = object, # must be an ExtensionClass or some such :( |
|
49 for t in mro: |
|
50 if t in registry: |
|
51 return registry[t](*args, **kw) |
|
52 else: |
|
53 return func(*args, **kw) |
|
54 try: |
|
55 wrapper.__name__ = func.__name__ |
|
56 except (TypeError, AttributeError): |
|
57 pass # Python 2.3 doesn't allow functions to be renamed |
|
58 |
|
59 def register(typ, func=None): |
|
60 if func is None: |
|
61 return lambda f: register(typ, f) |
|
62 registry[typ] = func |
|
63 return func |
|
64 |
|
65 wrapper.__dict__ = func.__dict__ |
|
66 wrapper.__doc__ = func.__doc__ |
|
67 wrapper.register = register |
|
68 return wrapper |
|
69 |
|
70 |
|
71 def walk_packages(path=None, prefix='', onerror=None): |
|
72 """Yields (module_loader, name, ispkg) for all modules recursively |
|
73 on path, or, if path is None, all accessible modules. |
|
74 |
|
75 'path' should be either None or a list of paths to look for |
|
76 modules in. |
|
77 |
|
78 'prefix' is a string to output on the front of every module name |
|
79 on output. |
|
80 |
|
81 Note that this function must import all *packages* (NOT all |
|
82 modules!) on the given path, in order to access the __path__ |
|
83 attribute to find submodules. |
|
84 |
|
85 'onerror' is a function which gets called with one argument (the |
|
86 name of the package which was being imported) if any exception |
|
87 occurs while trying to import a package. If no onerror function is |
|
88 supplied, ImportErrors are caught and ignored, while all other |
|
89 exceptions are propagated, terminating the search. |
|
90 |
|
91 Examples: |
|
92 |
|
93 # list all modules python can access |
|
94 walk_packages() |
|
95 |
|
96 # list all submodules of ctypes |
|
97 walk_packages(ctypes.__path__, ctypes.__name__+'.') |
|
98 """ |
|
99 |
|
100 def seen(p, m={}): |
|
101 if p in m: |
|
102 return True |
|
103 m[p] = True |
|
104 |
|
105 for importer, name, ispkg in iter_modules(path, prefix): |
|
106 yield importer, name, ispkg |
|
107 |
|
108 if ispkg: |
|
109 try: |
|
110 __import__(name) |
|
111 except ImportError: |
|
112 if onerror is not None: |
|
113 onerror(name) |
|
114 except Exception: |
|
115 if onerror is not None: |
|
116 onerror(name) |
|
117 else: |
|
118 raise |
|
119 else: |
|
120 path = getattr(sys.modules[name], '__path__', None) or [] |
|
121 |
|
122 # don't traverse path items we've seen before |
|
123 path = [p for p in path if not seen(p)] |
|
124 |
|
125 for item in walk_packages(path, name+'.', onerror): |
|
126 yield item |
|
127 |
|
128 |
|
129 def iter_modules(path=None, prefix=''): |
|
130 """Yields (module_loader, name, ispkg) for all submodules on path, |
|
131 or, if path is None, all top-level modules on sys.path. |
|
132 |
|
133 'path' should be either None or a list of paths to look for |
|
134 modules in. |
|
135 |
|
136 'prefix' is a string to output on the front of every module name |
|
137 on output. |
|
138 """ |
|
139 |
|
140 if path is None: |
|
141 importers = iter_importers() |
|
142 else: |
|
143 importers = map(get_importer, path) |
|
144 |
|
145 yielded = {} |
|
146 for i in importers: |
|
147 for name, ispkg in iter_importer_modules(i, prefix): |
|
148 if name not in yielded: |
|
149 yielded[name] = 1 |
|
150 yield i, name, ispkg |
|
151 |
|
152 |
|
153 #@simplegeneric |
|
154 def iter_importer_modules(importer, prefix=''): |
|
155 if not hasattr(importer, 'iter_modules'): |
|
156 return [] |
|
157 return importer.iter_modules(prefix) |
|
158 |
|
159 iter_importer_modules = simplegeneric(iter_importer_modules) |
|
160 |
|
161 |
|
162 class ImpImporter: |
|
163 """PEP 302 Importer that wraps Python's "classic" import algorithm |
|
164 |
|
165 ImpImporter(dirname) produces a PEP 302 importer that searches that |
|
166 directory. ImpImporter(None) produces a PEP 302 importer that searches |
|
167 the current sys.path, plus any modules that are frozen or built-in. |
|
168 |
|
169 Note that ImpImporter does not currently support being used by placement |
|
170 on sys.meta_path. |
|
171 """ |
|
172 |
|
173 def __init__(self, path=None): |
|
174 self.path = path |
|
175 |
|
176 def find_module(self, fullname, path=None): |
|
177 # Note: we ignore 'path' argument since it is only used via meta_path |
|
178 subname = fullname.split(".")[-1] |
|
179 if subname != fullname and self.path is None: |
|
180 return None |
|
181 if self.path is None: |
|
182 path = None |
|
183 else: |
|
184 path = [os.path.realpath(self.path)] |
|
185 try: |
|
186 file, filename, etc = imp.find_module(subname, path) |
|
187 except ImportError: |
|
188 return None |
|
189 return ImpLoader(fullname, file, filename, etc) |
|
190 |
|
191 def iter_modules(self, prefix=''): |
|
192 if self.path is None or not os.path.isdir(self.path): |
|
193 return |
|
194 |
|
195 yielded = {} |
|
196 import inspect |
|
197 |
|
198 filenames = os.listdir(self.path) |
|
199 filenames.sort() # handle packages before same-named modules |
|
200 |
|
201 for fn in filenames: |
|
202 modname = inspect.getmodulename(fn) |
|
203 if modname=='__init__' or modname in yielded: |
|
204 continue |
|
205 |
|
206 path = os.path.join(self.path, fn) |
|
207 ispkg = False |
|
208 |
|
209 if not modname and os.path.isdir(path) and '.' not in fn: |
|
210 modname = fn |
|
211 for fn in os.listdir(path): |
|
212 subname = inspect.getmodulename(fn) |
|
213 if subname=='__init__': |
|
214 ispkg = True |
|
215 break |
|
216 else: |
|
217 continue # not a package |
|
218 |
|
219 if modname and '.' not in modname: |
|
220 yielded[modname] = 1 |
|
221 yield prefix + modname, ispkg |
|
222 |
|
223 |
|
224 class ImpLoader: |
|
225 """PEP 302 Loader that wraps Python's "classic" import algorithm |
|
226 """ |
|
227 code = source = None |
|
228 |
|
229 def __init__(self, fullname, file, filename, etc): |
|
230 self.file = file |
|
231 self.filename = filename |
|
232 self.fullname = fullname |
|
233 self.etc = etc |
|
234 |
|
235 def load_module(self, fullname): |
|
236 self._reopen() |
|
237 try: |
|
238 mod = imp.load_module(fullname, self.file, self.filename, self.etc) |
|
239 finally: |
|
240 if self.file: |
|
241 self.file.close() |
|
242 # Note: we don't set __loader__ because we want the module to look |
|
243 # normal; i.e. this is just a wrapper for standard import machinery |
|
244 return mod |
|
245 |
|
246 def get_data(self, pathname): |
|
247 return open(pathname, "rb").read() |
|
248 |
|
249 def _reopen(self): |
|
250 if self.file and self.file.closed: |
|
251 mod_type = self.etc[2] |
|
252 if mod_type==imp.PY_SOURCE: |
|
253 self.file = open(self.filename, 'rU') |
|
254 elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): |
|
255 self.file = open(self.filename, 'rb') |
|
256 |
|
257 def _fix_name(self, fullname): |
|
258 if fullname is None: |
|
259 fullname = self.fullname |
|
260 elif fullname != self.fullname: |
|
261 raise ImportError("Loader for module %s cannot handle " |
|
262 "module %s" % (self.fullname, fullname)) |
|
263 return fullname |
|
264 |
|
265 def is_package(self, fullname): |
|
266 fullname = self._fix_name(fullname) |
|
267 return self.etc[2]==imp.PKG_DIRECTORY |
|
268 |
|
269 def get_code(self, fullname=None): |
|
270 fullname = self._fix_name(fullname) |
|
271 if self.code is None: |
|
272 mod_type = self.etc[2] |
|
273 if mod_type==imp.PY_SOURCE: |
|
274 source = self.get_source(fullname) |
|
275 self.code = compile(source, self.filename, 'exec') |
|
276 elif mod_type==imp.PY_COMPILED: |
|
277 self._reopen() |
|
278 try: |
|
279 self.code = read_code(self.file) |
|
280 finally: |
|
281 self.file.close() |
|
282 elif mod_type==imp.PKG_DIRECTORY: |
|
283 self.code = self._get_delegate().get_code() |
|
284 return self.code |
|
285 |
|
286 def get_source(self, fullname=None): |
|
287 fullname = self._fix_name(fullname) |
|
288 if self.source is None: |
|
289 mod_type = self.etc[2] |
|
290 if mod_type==imp.PY_SOURCE: |
|
291 self._reopen() |
|
292 try: |
|
293 self.source = self.file.read() |
|
294 finally: |
|
295 self.file.close() |
|
296 elif mod_type==imp.PY_COMPILED: |
|
297 if os.path.exists(self.filename[:-1]): |
|
298 f = open(self.filename[:-1], 'rU') |
|
299 self.source = f.read() |
|
300 f.close() |
|
301 elif mod_type==imp.PKG_DIRECTORY: |
|
302 self.source = self._get_delegate().get_source() |
|
303 return self.source |
|
304 |
|
305 |
|
306 def _get_delegate(self): |
|
307 return ImpImporter(self.filename).find_module('__init__') |
|
308 |
|
309 def get_filename(self, fullname=None): |
|
310 fullname = self._fix_name(fullname) |
|
311 mod_type = self.etc[2] |
|
312 if self.etc[2]==imp.PKG_DIRECTORY: |
|
313 return self._get_delegate().get_filename() |
|
314 elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): |
|
315 return self.filename |
|
316 return None |
|
317 |
|
318 |
|
319 try: |
|
320 import zipimport |
|
321 from zipimport import zipimporter |
|
322 |
|
323 def iter_zipimport_modules(importer, prefix=''): |
|
324 dirlist = zipimport._zip_directory_cache[importer.archive].keys() |
|
325 dirlist.sort() |
|
326 _prefix = importer.prefix |
|
327 plen = len(_prefix) |
|
328 yielded = {} |
|
329 import inspect |
|
330 for fn in dirlist: |
|
331 if not fn.startswith(_prefix): |
|
332 continue |
|
333 |
|
334 fn = fn[plen:].split(os.sep) |
|
335 |
|
336 if len(fn)==2 and fn[1].startswith('__init__.py'): |
|
337 if fn[0] not in yielded: |
|
338 yielded[fn[0]] = 1 |
|
339 yield fn[0], True |
|
340 |
|
341 if len(fn)!=1: |
|
342 continue |
|
343 |
|
344 modname = inspect.getmodulename(fn[0]) |
|
345 if modname=='__init__': |
|
346 continue |
|
347 |
|
348 if modname and '.' not in modname and modname not in yielded: |
|
349 yielded[modname] = 1 |
|
350 yield prefix + modname, False |
|
351 |
|
352 iter_importer_modules.register(zipimporter, iter_zipimport_modules) |
|
353 |
|
354 except ImportError: |
|
355 pass |
|
356 |
|
357 |
|
358 def get_importer(path_item): |
|
359 """Retrieve a PEP 302 importer for the given path item |
|
360 |
|
361 The returned importer is cached in sys.path_importer_cache |
|
362 if it was newly created by a path hook. |
|
363 |
|
364 If there is no importer, a wrapper around the basic import |
|
365 machinery is returned. This wrapper is never inserted into |
|
366 the importer cache (None is inserted instead). |
|
367 |
|
368 The cache (or part of it) can be cleared manually if a |
|
369 rescan of sys.path_hooks is necessary. |
|
370 """ |
|
371 try: |
|
372 importer = sys.path_importer_cache[path_item] |
|
373 except KeyError: |
|
374 for path_hook in sys.path_hooks: |
|
375 try: |
|
376 importer = path_hook(path_item) |
|
377 break |
|
378 except ImportError: |
|
379 pass |
|
380 else: |
|
381 importer = None |
|
382 sys.path_importer_cache.setdefault(path_item, importer) |
|
383 |
|
384 if importer is None: |
|
385 try: |
|
386 importer = ImpImporter(path_item) |
|
387 except ImportError: |
|
388 importer = None |
|
389 return importer |
|
390 |
|
391 |
|
392 def iter_importers(fullname=""): |
|
393 """Yield PEP 302 importers for the given module name |
|
394 |
|
395 If fullname contains a '.', the importers will be for the package |
|
396 containing fullname, otherwise they will be importers for sys.meta_path, |
|
397 sys.path, and Python's "classic" import machinery, in that order. If |
|
398 the named module is in a package, that package is imported as a side |
|
399 effect of invoking this function. |
|
400 |
|
401 Non PEP 302 mechanisms (e.g. the Windows registry) used by the |
|
402 standard import machinery to find files in alternative locations |
|
403 are partially supported, but are searched AFTER sys.path. Normally, |
|
404 these locations are searched BEFORE sys.path, preventing sys.path |
|
405 entries from shadowing them. |
|
406 |
|
407 For this to cause a visible difference in behaviour, there must |
|
408 be a module or package name that is accessible via both sys.path |
|
409 and one of the non PEP 302 file system mechanisms. In this case, |
|
410 the emulation will find the former version, while the builtin |
|
411 import mechanism will find the latter. |
|
412 |
|
413 Items of the following types can be affected by this discrepancy: |
|
414 imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY |
|
415 """ |
|
416 if fullname.startswith('.'): |
|
417 raise ImportError("Relative module names not supported") |
|
418 if '.' in fullname: |
|
419 # Get the containing package's __path__ |
|
420 pkg = '.'.join(fullname.split('.')[:-1]) |
|
421 if pkg not in sys.modules: |
|
422 __import__(pkg) |
|
423 path = getattr(sys.modules[pkg], '__path__', None) or [] |
|
424 else: |
|
425 for importer in sys.meta_path: |
|
426 yield importer |
|
427 path = sys.path |
|
428 for item in path: |
|
429 yield get_importer(item) |
|
430 if '.' not in fullname: |
|
431 yield ImpImporter() |
|
432 |
|
433 def get_loader(module_or_name): |
|
434 """Get a PEP 302 "loader" object for module_or_name |
|
435 |
|
436 If the module or package is accessible via the normal import |
|
437 mechanism, a wrapper around the relevant part of that machinery |
|
438 is returned. Returns None if the module cannot be found or imported. |
|
439 If the named module is not already imported, its containing package |
|
440 (if any) is imported, in order to establish the package __path__. |
|
441 |
|
442 This function uses iter_importers(), and is thus subject to the same |
|
443 limitations regarding platform-specific special import locations such |
|
444 as the Windows registry. |
|
445 """ |
|
446 if module_or_name in sys.modules: |
|
447 module_or_name = sys.modules[module_or_name] |
|
448 if isinstance(module_or_name, ModuleType): |
|
449 module = module_or_name |
|
450 loader = getattr(module, '__loader__', None) |
|
451 if loader is not None: |
|
452 return loader |
|
453 fullname = module.__name__ |
|
454 else: |
|
455 fullname = module_or_name |
|
456 return find_loader(fullname) |
|
457 |
|
458 def find_loader(fullname): |
|
459 """Find a PEP 302 "loader" object for fullname |
|
460 |
|
461 If fullname contains dots, path must be the containing package's __path__. |
|
462 Returns None if the module cannot be found or imported. This function uses |
|
463 iter_importers(), and is thus subject to the same limitations regarding |
|
464 platform-specific special import locations such as the Windows registry. |
|
465 """ |
|
466 for importer in iter_importers(fullname): |
|
467 loader = importer.find_module(fullname) |
|
468 if loader is not None: |
|
469 return loader |
|
470 |
|
471 return None |
|
472 |
|
473 |
|
474 def extend_path(path, name): |
|
475 """Extend a package's path. |
|
476 |
|
477 Intended use is to place the following code in a package's __init__.py: |
|
478 |
|
479 from pkgutil import extend_path |
|
480 __path__ = extend_path(__path__, __name__) |
|
481 |
|
482 This will add to the package's __path__ all subdirectories of |
|
483 directories on sys.path named after the package. This is useful |
|
484 if one wants to distribute different parts of a single logical |
|
485 package as multiple directories. |
|
486 |
|
487 It also looks for *.pkg files beginning where * matches the name |
|
488 argument. This feature is similar to *.pth files (see site.py), |
|
489 except that it doesn't special-case lines starting with 'import'. |
|
490 A *.pkg file is trusted at face value: apart from checking for |
|
491 duplicates, all entries found in a *.pkg file are added to the |
|
492 path, regardless of whether they are exist the filesystem. (This |
|
493 is a feature.) |
|
494 |
|
495 If the input path is not a list (as is the case for frozen |
|
496 packages) it is returned unchanged. The input path is not |
|
497 modified; an extended copy is returned. Items are only appended |
|
498 to the copy at the end. |
|
499 |
|
500 It is assumed that sys.path is a sequence. Items of sys.path that |
|
501 are not (unicode or 8-bit) strings referring to existing |
|
502 directories are ignored. Unicode items of sys.path that cause |
|
503 errors when used as filenames may cause this function to raise an |
|
504 exception (in line with os.path.isdir() behavior). |
|
505 """ |
|
506 |
|
507 if not isinstance(path, list): |
|
508 # This could happen e.g. when this is called from inside a |
|
509 # frozen package. Return the path unchanged in that case. |
|
510 return path |
|
511 |
|
512 pname = os.path.join(*name.split('.')) # Reconstitute as relative path |
|
513 # Just in case os.extsep != '.' |
|
514 sname = os.extsep.join(name.split('.')) |
|
515 sname_pkg = sname + os.extsep + "pkg" |
|
516 init_py = "__init__" + os.extsep + "py" |
|
517 |
|
518 path = path[:] # Start with a copy of the existing path |
|
519 |
|
520 for dir in sys.path: |
|
521 if not isinstance(dir, basestring) or not os.path.isdir(dir): |
|
522 continue |
|
523 subdir = os.path.join(dir, pname) |
|
524 # XXX This may still add duplicate entries to path on |
|
525 # case-insensitive filesystems |
|
526 initfile = os.path.join(subdir, init_py) |
|
527 if subdir not in path and os.path.isfile(initfile): |
|
528 path.append(subdir) |
|
529 # XXX Is this the right thing for subpackages like zope.app? |
|
530 # It looks for a file named "zope.app.pkg" |
|
531 pkgfile = os.path.join(dir, sname_pkg) |
|
532 if os.path.isfile(pkgfile): |
|
533 try: |
|
534 f = open(pkgfile) |
|
535 except IOError, msg: |
|
536 sys.stderr.write("Can't open %s: %s\n" % |
|
537 (pkgfile, msg)) |
|
538 else: |
|
539 for line in f: |
|
540 line = line.rstrip('\n') |
|
541 if not line or line.startswith('#'): |
|
542 continue |
|
543 path.append(line) # Don't check for existence! |
|
544 f.close() |
|
545 |
|
546 return path |