1 """Utilities to support packages."""

     2 

     3 # NOTE: This module must remain compatible with Python 2.3, as it is shared

     4 # by setuptools for distribution with Python 2.3 and up.

     5 

     6 import os

     7 import sys

     8 import imp

     9 import os.path

    10 from types import ModuleType

    11 

    12 __all__ = [

    13     'get_importer', 'iter_importers', 'get_loader', 'find_loader',

    14     'walk_packages', 'iter_modules',

    15     'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',

    16 ]

    17 

    18 def read_code(stream):

    19     # This helper is needed in order for the PEP 302 emulation to

    20     # correctly handle compiled files

    21     import marshal

    22 

    23     magic = stream.read(4)

    24     if magic != imp.get_magic():

    25         return None

    26 

    27     stream.read(4) # Skip timestamp

    28     return marshal.load(stream)

    29 

    30 

    31 def simplegeneric(func):

    32     """Make a trivial single-dispatch generic function"""

    33     registry = {}

    34     def wrapper(*args, **kw):

    35         ob = args[0]

    36         try:

    37             cls = ob.__class__

    38         except AttributeError:

    39             cls = type(ob)

    40         try:

    41             mro = cls.__mro__

    42         except AttributeError:

    43             try:

    44                 class cls(cls, object):

    45                     pass

    46                 mro = cls.__mro__[1:]

    47             except TypeError:

    48                 mro = object,   # must be an ExtensionClass or some such  :(

    49         for t in mro:

    50             if t in registry:

    51                 return registry[t](*args, **kw)

    52         else:

    53             return func(*args, **kw)

    54     try:

    55         wrapper.__name__ = func.__name__

    56     except (TypeError, AttributeError):

    57         pass    # Python 2.3 doesn't allow functions to be renamed

    58 

    59     def register(typ, func=None):

    60         if func is None:

    61             return lambda f: register(typ, f)

    62         registry[typ] = func

    63         return func

    64 

    65     wrapper.__dict__ = func.__dict__

    66     wrapper.__doc__ = func.__doc__

    67     wrapper.register = register

    68     return wrapper

    69 

    70 

    71 def walk_packages(path=None, prefix='', onerror=None):

    72     """Yields (module_loader, name, ispkg) for all modules recursively

    73     on path, or, if path is None, all accessible modules.

    74 

    75     'path' should be either None or a list of paths to look for

    76     modules in.

    77 

    78     'prefix' is a string to output on the front of every module name

    79     on output.

    80 

    81     Note that this function must import all *packages* (NOT all

    82     modules!) on the given path, in order to access the __path__

    83     attribute to find submodules.

    84 

    85     'onerror' is a function which gets called with one argument (the

    86     name of the package which was being imported) if any exception

    87     occurs while trying to import a package.  If no onerror function is

    88     supplied, ImportErrors are caught and ignored, while all other

    89     exceptions are propagated, terminating the search.

    90 

    91     Examples:

    92 

    93     # list all modules python can access

    94     walk_packages()

    95 

    96     # list all submodules of ctypes

    97     walk_packages(ctypes.__path__, ctypes.__name__+'.')

    98     """

    99 

   100     def seen(p, m={}):

   101         if p in m:

   102             return True

   103         m[p] = True

   104 

   105     for importer, name, ispkg in iter_modules(path, prefix):

   106         yield importer, name, ispkg

   107 

   108         if ispkg:

   109             try:

   110                 __import__(name)

   111             except ImportError:

   112                 if onerror is not None:

   113                     onerror(name)

   114             except Exception:

   115                 if onerror is not None:

   116                     onerror(name)

   117                 else:

   118                     raise

   119             else:

   120                 path = getattr(sys.modules[name], '__path__', None) or []

   121 

   122                 # don't traverse path items we've seen before

   123                 path = [p for p in path if not seen(p)]

   124 

   125                 for item in walk_packages(path, name+'.', onerror):

   126                     yield item

   127 

   128 

   129 def iter_modules(path=None, prefix=''):

   130     """Yields (module_loader, name, ispkg) for all submodules on path,

   131     or, if path is None, all top-level modules on sys.path.

   132 

   133     'path' should be either None or a list of paths to look for

   134     modules in.

   135 

   136     'prefix' is a string to output on the front of every module name

   137     on output.

   138     """

   139 

   140     if path is None:

   141         importers = iter_importers()

   142     else:

   143         importers = map(get_importer, path)

   144 

   145     yielded = {}

   146     for i in importers:

   147         for name, ispkg in iter_importer_modules(i, prefix):

   148             if name not in yielded:

   149                 yielded[name] = 1

   150                 yield i, name, ispkg

   151 

   152 

   153 #@simplegeneric

   154 def iter_importer_modules(importer, prefix=''):

   155     if not hasattr(importer, 'iter_modules'):

   156         return []

   157     return importer.iter_modules(prefix)

   158 

   159 iter_importer_modules = simplegeneric(iter_importer_modules)

   160 

   161 

   162 class ImpImporter:

   163     """PEP 302 Importer that wraps Python's "classic" import algorithm

   164 

   165     ImpImporter(dirname) produces a PEP 302 importer that searches that

   166     directory.  ImpImporter(None) produces a PEP 302 importer that searches

   167     the current sys.path, plus any modules that are frozen or built-in.

   168 

   169     Note that ImpImporter does not currently support being used by placement

   170     on sys.meta_path.

   171     """

   172 

   173     def __init__(self, path=None):

   174         self.path = path

   175 

   176     def find_module(self, fullname, path=None):

   177         # Note: we ignore 'path' argument since it is only used via meta_path

   178         subname = fullname.split(".")[-1]

   179         if subname != fullname and self.path is None:

   180             return None

   181         if self.path is None:

   182             path = None

   183         else:

   184             path = [os.path.realpath(self.path)]

   185         try:

   186             file, filename, etc = imp.find_module(subname, path)

   187         except ImportError:

   188             return None

   189         return ImpLoader(fullname, file, filename, etc)

   190 

   191     def iter_modules(self, prefix=''):

   192         if self.path is None or not os.path.isdir(self.path):

   193             return

   194 

   195         yielded = {}

   196         import inspect

   197 

   198         filenames = os.listdir(self.path)

   199         filenames.sort()  # handle packages before same-named modules

   200 

   201         for fn in filenames:

   202             modname = inspect.getmodulename(fn)

   203             if modname=='__init__' or modname in yielded:

   204                 continue

   205 

   206             path = os.path.join(self.path, fn)

   207             ispkg = False

   208 

   209             if not modname and os.path.isdir(path) and '.' not in fn:

   210                 modname = fn

   211                 for fn in os.listdir(path):

   212                     subname = inspect.getmodulename(fn)

   213                     if subname=='__init__':

   214                         ispkg = True

   215                         break

   216                 else:

   217                     continue    # not a package

   218 

   219             if modname and '.' not in modname:

   220                 yielded[modname] = 1

   221                 yield prefix + modname, ispkg

   222 

   223 

   224 class ImpLoader:

   225     """PEP 302 Loader that wraps Python's "classic" import algorithm

   226     """

   227     code = source = None

   228 

   229     def __init__(self, fullname, file, filename, etc):

   230         self.file = file

   231         self.filename = filename

   232         self.fullname = fullname

   233         self.etc = etc

   234 

   235     def load_module(self, fullname):

   236         self._reopen()

   237         try:

   238             mod = imp.load_module(fullname, self.file, self.filename, self.etc)

   239         finally:

   240             if self.file:

   241                 self.file.close()

   242         # Note: we don't set __loader__ because we want the module to look

   243         # normal; i.e. this is just a wrapper for standard import machinery

   244         return mod

   245 

   246     def get_data(self, pathname):

   247         return open(pathname, "rb").read()

   248 

   249     def _reopen(self):

   250         if self.file and self.file.closed:

   251             mod_type = self.etc[2]

   252             if mod_type==imp.PY_SOURCE:

   253                 self.file = open(self.filename, 'rU')

   254             elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):

   255                 self.file = open(self.filename, 'rb')

   256 

   257     def _fix_name(self, fullname):

   258         if fullname is None:

   259             fullname = self.fullname

   260         elif fullname != self.fullname:

   261             raise ImportError("Loader for module %s cannot handle "

   262                               "module %s" % (self.fullname, fullname))

   263         return fullname

   264 

   265     def is_package(self, fullname):

   266         fullname = self._fix_name(fullname)

   267         return self.etc[2]==imp.PKG_DIRECTORY

   268 

   269     def get_code(self, fullname=None):

   270         fullname = self._fix_name(fullname)

   271         if self.code is None:

   272             mod_type = self.etc[2]

   273             if mod_type==imp.PY_SOURCE:

   274                 source = self.get_source(fullname)

   275                 self.code = compile(source, self.filename, 'exec')

   276             elif mod_type==imp.PY_COMPILED:

   277                 self._reopen()

   278                 try:

   279                     self.code = read_code(self.file)

   280                 finally:

   281                     self.file.close()

   282             elif mod_type==imp.PKG_DIRECTORY:

   283                 self.code = self._get_delegate().get_code()

   284         return self.code

   285 

   286     def get_source(self, fullname=None):

   287         fullname = self._fix_name(fullname)

   288         if self.source is None:

   289             mod_type = self.etc[2]

   290             if mod_type==imp.PY_SOURCE:

   291                 self._reopen()

   292                 try:

   293                     self.source = self.file.read()

   294                 finally:

   295                     self.file.close()

   296             elif mod_type==imp.PY_COMPILED:

   297                 if os.path.exists(self.filename[:-1]):

   298                     f = open(self.filename[:-1], 'rU')

   299                     self.source = f.read()

   300                     f.close()

   301             elif mod_type==imp.PKG_DIRECTORY:

   302                 self.source = self._get_delegate().get_source()

   303         return self.source

   304 

   305 

   306     def _get_delegate(self):

   307         return ImpImporter(self.filename).find_module('__init__')

   308 

   309     def get_filename(self, fullname=None):

   310         fullname = self._fix_name(fullname)

   311         mod_type = self.etc[2]

   312         if self.etc[2]==imp.PKG_DIRECTORY:

   313             return self._get_delegate().get_filename()

   314         elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):

   315             return self.filename

   316         return None

   317 

   318 

   319 try:

   320     import zipimport

   321     from zipimport import zipimporter

   322 

   323     def iter_zipimport_modules(importer, prefix=''):

   324         dirlist = zipimport._zip_directory_cache[importer.archive].keys()

   325         dirlist.sort()

   326         _prefix = importer.prefix

   327         plen = len(_prefix)

   328         yielded = {}

   329         import inspect

   330         for fn in dirlist:

   331             if not fn.startswith(_prefix):

   332                 continue

   333 

   334             fn = fn[plen:].split(os.sep)

   335 

   336             if len(fn)==2 and fn[1].startswith('__init__.py'):

   337                 if fn[0] not in yielded:

   338                     yielded[fn[0]] = 1

   339                     yield fn[0], True

   340 

   341             if len(fn)!=1:

   342                 continue

   343 

   344             modname = inspect.getmodulename(fn[0])

   345             if modname=='__init__':

   346                 continue

   347 

   348             if modname and '.' not in modname and modname not in yielded:

   349                 yielded[modname] = 1

   350                 yield prefix + modname, False

   351 

   352     iter_importer_modules.register(zipimporter, iter_zipimport_modules)

   353 

   354 except ImportError:

   355     pass

   356 

   357 

   358 def get_importer(path_item):

   359     """Retrieve a PEP 302 importer for the given path item

   360 

   361     The returned importer is cached in sys.path_importer_cache

   362     if it was newly created by a path hook.

   363 

   364     If there is no importer, a wrapper around the basic import

   365     machinery is returned. This wrapper is never inserted into

   366     the importer cache (None is inserted instead).

   367 

   368     The cache (or part of it) can be cleared manually if a

   369     rescan of sys.path_hooks is necessary.

   370     """

   371     try:

   372         importer = sys.path_importer_cache[path_item]

   373     except KeyError:

   374         for path_hook in sys.path_hooks:

   375             try:

   376                 importer = path_hook(path_item)

   377                 break

   378             except ImportError:

   379                 pass

   380         else:

   381             importer = None

   382         sys.path_importer_cache.setdefault(path_item, importer)

   383 

   384     if importer is None:

   385         try:

   386             importer = ImpImporter(path_item)

   387         except ImportError:

   388             importer = None

   389     return importer

   390 

   391 

   392 def iter_importers(fullname=""):

   393     """Yield PEP 302 importers for the given module name

   394 

   395     If fullname contains a '.', the importers will be for the package

   396     containing fullname, otherwise they will be importers for sys.meta_path,

   397     sys.path, and Python's "classic" import machinery, in that order.  If

   398     the named module is in a package, that package is imported as a side

   399     effect of invoking this function.

   400 

   401     Non PEP 302 mechanisms (e.g. the Windows registry) used by the

   402     standard import machinery to find files in alternative locations

   403     are partially supported, but are searched AFTER sys.path. Normally,

   404     these locations are searched BEFORE sys.path, preventing sys.path

   405     entries from shadowing them.

   406 

   407     For this to cause a visible difference in behaviour, there must

   408     be a module or package name that is accessible via both sys.path

   409     and one of the non PEP 302 file system mechanisms. In this case,

   410     the emulation will find the former version, while the builtin

   411     import mechanism will find the latter.

   412 

   413     Items of the following types can be affected by this discrepancy:

   414         imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY

   415     """

   416     if fullname.startswith('.'):

   417         raise ImportError("Relative module names not supported")

   418     if '.' in fullname:

   419         # Get the containing package's __path__

   420         pkg = '.'.join(fullname.split('.')[:-1])

   421         if pkg not in sys.modules:

   422             __import__(pkg)

   423         path = getattr(sys.modules[pkg], '__path__', None) or []

   424     else:

   425         for importer in sys.meta_path:

   426             yield importer

   427         path = sys.path

   428     for item in path:

   429         yield get_importer(item)

   430     if '.' not in fullname:

   431         yield ImpImporter()

   432 

   433 def get_loader(module_or_name):

   434     """Get a PEP 302 "loader" object for module_or_name

   435 

   436     If the module or package is accessible via the normal import

   437     mechanism, a wrapper around the relevant part of that machinery

   438     is returned.  Returns None if the module cannot be found or imported.

   439     If the named module is not already imported, its containing package

   440     (if any) is imported, in order to establish the package __path__.

   441 

   442     This function uses iter_importers(), and is thus subject to the same

   443     limitations regarding platform-specific special import locations such

   444     as the Windows registry.

   445     """

   446     if module_or_name in sys.modules:

   447         module_or_name = sys.modules[module_or_name]

   448     if isinstance(module_or_name, ModuleType):

   449         module = module_or_name

   450         loader = getattr(module, '__loader__', None)

   451         if loader is not None:

   452             return loader

   453         fullname = module.__name__

   454     else:

   455         fullname = module_or_name

   456     return find_loader(fullname)

   457 

   458 def find_loader(fullname):

   459     """Find a PEP 302 "loader" object for fullname

   460 

   461     If fullname contains dots, path must be the containing package's __path__.

   462     Returns None if the module cannot be found or imported. This function uses

   463     iter_importers(), and is thus subject to the same limitations regarding

   464     platform-specific special import locations such as the Windows registry.

   465     """

   466     for importer in iter_importers(fullname):

   467         loader = importer.find_module(fullname)

   468         if loader is not None:

   469             return loader

   470 

   471     return None

   472 

   473 

   474 def extend_path(path, name):

   475     """Extend a package's path.

   476 

   477     Intended use is to place the following code in a package's __init__.py:

   478 

   479         from pkgutil import extend_path

   480         __path__ = extend_path(__path__, __name__)

   481 

   482     This will add to the package's __path__ all subdirectories of

   483     directories on sys.path named after the package.  This is useful

   484     if one wants to distribute different parts of a single logical

   485     package as multiple directories.

   486 

   487     It also looks for *.pkg files beginning where * matches the name

   488     argument.  This feature is similar to *.pth files (see site.py),

   489     except that it doesn't special-case lines starting with 'import'.

   490     A *.pkg file is trusted at face value: apart from checking for

   491     duplicates, all entries found in a *.pkg file are added to the

   492     path, regardless of whether they are exist the filesystem.  (This

   493     is a feature.)

   494 

   495     If the input path is not a list (as is the case for frozen

   496     packages) it is returned unchanged.  The input path is not

   497     modified; an extended copy is returned.  Items are only appended

   498     to the copy at the end.

   499 

   500     It is assumed that sys.path is a sequence.  Items of sys.path that

   501     are not (unicode or 8-bit) strings referring to existing

   502     directories are ignored.  Unicode items of sys.path that cause

   503     errors when used as filenames may cause this function to raise an

   504     exception (in line with os.path.isdir() behavior).

   505     """

   506 

   507     if not isinstance(path, list):

   508         # This could happen e.g. when this is called from inside a

   509         # frozen package.  Return the path unchanged in that case.

   510         return path

   511 

   512     pname = os.path.join(*name.split('.')) # Reconstitute as relative path

   513     # Just in case os.extsep != '.'

   514     sname = os.extsep.join(name.split('.'))

   515     sname_pkg = sname + os.extsep + "pkg"

   516     init_py = "__init__" + os.extsep + "py"

   517 

   518     path = path[:] # Start with a copy of the existing path

   519 

   520     for dir in sys.path:

   521         if not isinstance(dir, basestring) or not os.path.isdir(dir):

   522             continue

   523         subdir = os.path.join(dir, pname)

   524         # XXX This may still add duplicate entries to path on

   525         # case-insensitive filesystems

   526         initfile = os.path.join(subdir, init_py)

   527         if subdir not in path and os.path.isfile(initfile):

   528             path.append(subdir)

   529         # XXX Is this the right thing for subpackages like zope.app?

   530         # It looks for a file named "zope.app.pkg"

   531         pkgfile = os.path.join(dir, sname_pkg)

   532         if os.path.isfile(pkgfile):

   533             try:

   534                 f = open(pkgfile)

   535             except IOError, msg:

   536                 sys.stderr.write("Can't open %s: %s\n" %

   537                                  (pkgfile, msg))

   538             else:

   539                 for line in f:

   540                     line = line.rstrip('\n')

   541                     if not line or line.startswith('#'):

   542                         continue

   543                     path.append(line) # Don't check for existence!

   544                 f.close()

   545 

   546     return path

   547 

   548 def get_data(package, resource):

   549     """Get a resource from a package.

   550 

   551     This is a wrapper round the PEP 302 loader get_data API. The package

   552     argument should be the name of a package, in standard module format

   553     (foo.bar). The resource argument should be in the form of a relative

   554     filename, using '/' as the path separator. The parent directory name '..'

   555     is not allowed, and nor is a rooted name (starting with a '/').

   556 

   557     The function returns a binary string, which is the contents of the

   558     specified resource.

   559 

   560     For packages located in the filesystem, which have already been imported,

   561     this is the rough equivalent of

   562 

   563         d = os.path.dirname(sys.modules[package].__file__)

   564         data = open(os.path.join(d, resource), 'rb').read()

   565 

   566     If the package cannot be located or loaded, or it uses a PEP 302 loader

   567     which does not support get_data(), then None is returned.

   568     """

   569 

   570     loader = get_loader(package)

   571     if loader is None or not hasattr(loader, 'get_data'):

   572         return None

   573     mod = sys.modules.get(package) or loader.load_module(package)

   574     if mod is None or not hasattr(mod, '__file__'):

   575         return None

   576 

   577     # Modify the resource name to be compatible with the loader.get_data

   578     # signature - an os.path format "filename" starting with the dirname of

   579     # the package's __file__

   580     parts = resource.split('/')

   581     parts.insert(0, os.path.dirname(mod.__file__))

   582     resource_name = os.path.join(*parts)

   583     return loader.get_data(resource_name)