1 # Copyright (C) 2005, Giovanni Bajo

     2 # Based on previous work under copyright (c) 2002 McMillan Enterprises, Inc.

     3 #

     4 # This program is free software; you can redistribute it and/or

     5 # modify it under the terms of the GNU General Public License

     6 # as published by the Free Software Foundation; either version 2

     7 # of the License, or (at your option) any later version.

     8 #

     9 # This program is distributed in the hope that it will be useful,

    10 # but WITHOUT ANY WARRANTY; without even the implied warranty of

    11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    12 # GNU General Public License for more details.

    13 #

    14 # You should have received a copy of the GNU General Public License

    15 # along with this program; if not, write to the Free Software

    16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA

    17 import sys, string, os, imp, marshal

    18 

    19 #=======================Owners==========================#

    20 # An Owner does imports from a particular piece of turf

    21 # That is, there's an Owner for each thing on sys.path

    22 # There are owners for directories and .pyz files.

    23 # There could be owners for zip files, or even URLs.

    24 # Note that they replace the string in sys.path,

    25 # but str(sys.path[n]) should yield the original string.

    26 

    27 STRINGTYPE = type('')

    28 

    29 class Owner:

    30     def __init__(self, path):

    31         self.path = path

    32     def __str__(self):

    33         return self.path

    34     def getmod(self, nm):

    35         return None

    36 

    37 class DirOwner(Owner):

    38     def __init__(self, path):

    39         if path == '':

    40             path = os.getcwd()

    41         if not os.path.isdir(path):

    42             raise ValueError, "%s is not a directory" % path

    43         Owner.__init__(self, path)

    44     def getmod(self, nm, getsuffixes=imp.get_suffixes, loadco=marshal.loads):

    45         pth =  os.path.join(self.path, nm)

    46         possibles = [(pth, 0, None)]

    47         if os.path.isdir(pth):

    48             possibles.insert(0, (os.path.join(pth, '__init__'), 1, pth))

    49         py = pyc = None

    50         for pth, ispkg, pkgpth in possibles:

    51             for ext, mode, typ in getsuffixes():

    52                 attempt = pth+ext

    53                 try:

    54                     st = os.stat(attempt)

    55                 except:

    56                     pass

    57                 else:

    58                     if typ == imp.C_EXTENSION:

    59                         return ExtensionModule(nm, attempt)

    60                     elif typ == imp.PY_SOURCE:

    61                         py = (attempt, st)

    62                     else:

    63                         pyc = (attempt, st)

    64             if py or pyc:

    65                 break

    66         if py is None and pyc is None:

    67             return None

    68         while 1:

    69             if pyc is None or py and pyc[1][8] < py[1][8]:

    70                 try:

    71                     co = compile(open(py[0], 'r').read()+'\n', py[0], 'exec')

    72                     if __debug__:

    73                         pth = py[0] + 'c'

    74                     else:

    75                         pth = py[0] + 'o'

    76                     break

    77                 except SyntaxError, e:

    78                     print "Syntax error in", py[0]

    79                     print e.args

    80                     raise

    81             elif pyc:

    82                 stuff = open(pyc[0], 'rb').read()

    83                 try:

    84                     co = loadco(stuff[8:])

    85                     pth = pyc[0]

    86                     break

    87                 except (ValueError, EOFError):

    88                     print "W: bad .pyc found (%s)" % pyc[0]

    89                     pyc = None

    90             else:

    91                 return None

    92         if not os.path.isabs(pth):

    93             pth = os.path.abspath(pth)

    94         if ispkg:

    95             mod = PkgModule(nm, pth, co)

    96         else:

    97             mod = PyModule(nm, pth, co)

    98         return mod

    99 

   100 class PYZOwner(Owner):

   101     def __init__(self, path):

   102         import archive

   103         self.pyz = archive.ZlibArchive(path)

   104         Owner.__init__(self, path)

   105     def getmod(self, nm):

   106         rslt = self.pyz.extract(nm)

   107         if rslt:

   108             ispkg, co = rslt

   109         if ispkg:

   110             return PkgInPYZModule(nm, co, self)

   111         return PyModule(nm, self.path, co)

   112 

   113 _globalownertypes = [

   114     DirOwner,

   115     PYZOwner,

   116     Owner,

   117 ]

   118 

   119 #===================Import Directors====================================#

   120 # ImportDirectors live on the metapath

   121 # There's one for builtins, one for frozen modules, and one for sys.path

   122 # Windows gets one for modules gotten from the Registry

   123 # There should be one for Frozen modules

   124 # Mac would have them for PY_RESOURCE modules etc.

   125 # A generalization of Owner - their concept of "turf" is broader

   126 

   127 class ImportDirector(Owner):

   128     pass

   129 class BuiltinImportDirector(ImportDirector):

   130     def __init__(self):

   131         self.path = 'Builtins'

   132     def getmod(self, nm, isbuiltin=imp.is_builtin):

   133         if isbuiltin(nm):

   134             return BuiltinModule(nm)

   135         return None

   136 class FrozenImportDirector(ImportDirector):

   137     def __init__(self):

   138         self.path = 'FrozenModules'

   139     def getmod(self, nm, isfrozen=imp.is_frozen):

   140         if isfrozen(nm):

   141             return FrozenModule(nm)

   142         return None

   143 class RegistryImportDirector(ImportDirector):

   144     # for Windows only

   145     def __init__(self):

   146         self.path = "WindowsRegistry"

   147         self.map = {}

   148         try:

   149             import win32api

   150             import win32con

   151         except ImportError:

   152             pass

   153         else:

   154             subkey = r"Software\Python\PythonCore\%s\Modules" % sys.winver

   155             for root in (win32con.HKEY_CURRENT_USER, win32con.HKEY_LOCAL_MACHINE):

   156                 try:

   157                     #hkey = win32api.RegOpenKeyEx(root, subkey, 0, win32con.KEY_ALL_ACCESS)

   158                     hkey = win32api.RegOpenKeyEx(root, subkey, 0, win32con.KEY_READ)

   159                 except:

   160                     pass

   161                 else:

   162                     numsubkeys, numvalues, lastmodified = win32api.RegQueryInfoKey(hkey)

   163                     for i in range(numsubkeys):

   164                         subkeyname = win32api.RegEnumKey(hkey, i)

   165                         #hskey = win32api.RegOpenKeyEx(hkey, subkeyname, 0, win32con.KEY_ALL_ACCESS)

   166                         hskey = win32api.RegOpenKeyEx(hkey, subkeyname, 0, win32con.KEY_READ)

   167                         val = win32api.RegQueryValueEx(hskey, '')

   168                         desc = getDescr(val[0])

   169                         #print " RegistryImportDirector got %s %s" % (val[0], desc)  #XXX

   170                         self.map[subkeyname] = (val[0], desc)

   171                         hskey.Close()

   172                     hkey.Close()

   173                     break

   174     def getmod(self, nm):

   175         stuff = self.map.get(nm)

   176         if stuff:

   177             fnm, (suffix, mode, typ) = stuff

   178             if typ == imp.C_EXTENSION:

   179                 return ExtensionModule(nm, fnm)

   180             elif typ == imp.PY_SOURCE:

   181                 try:

   182                     co = compile(open(fnm, 'r').read()+'\n', fnm, 'exec')

   183                 except SyntaxError, e:

   184                     print "Invalid syntax in %s" % py[0]

   185                     print e.args

   186                     raise

   187             else:

   188                 stuff = open(fnm, 'rb').read()

   189                 co = loadco(stuff[8:])

   190             return PyModule(nm, fnm, co)

   191         return None

   192 class PathImportDirector(ImportDirector):

   193     def __init__(self, pathlist=None, importers=None, ownertypes=None):

   194         if pathlist is None:

   195             self.path = sys.path

   196         else:

   197             self.path = pathlist

   198         if ownertypes == None:

   199             self.ownertypes = _globalownertypes

   200         else:

   201             self.ownertypes = ownertypes

   202         if importers:

   203             self.shadowpath = importers

   204         else:

   205             self.shadowpath = {}

   206         self.inMakeOwner = 0

   207         self.building = {}

   208     def getmod(self, nm):

   209         mod = None

   210         for thing in self.path:

   211             if type(thing) is STRINGTYPE:

   212                 owner = self.shadowpath.get(thing, -1)

   213                 if owner == -1:

   214                     owner = self.shadowpath[thing] = self.makeOwner(thing)

   215                 if owner:

   216                     mod = owner.getmod(nm)

   217             else:

   218                 mod = thing.getmod(nm)

   219             if mod:

   220                 break

   221         return mod

   222     def makeOwner(self, path):

   223         if self.building.get(path):

   224             return None

   225         self.building[path] = 1

   226         owner = None

   227         for klass in self.ownertypes:

   228             try:

   229                 # this may cause an import, which may cause recursion

   230                 # hence the protection

   231                 owner = klass(path)

   232             except:

   233                 pass

   234             else:

   235                 break

   236         del self.building[path]

   237         return owner

   238 

   239 

   240 def getDescr(fnm):

   241     ext = os.path.splitext(fnm)[1]

   242     for (suffix, mode, typ) in imp.get_suffixes():

   243         if suffix == ext:

   244             return (suffix, mode, typ)

   245 

   246 #=================Import Tracker============================#

   247 # This one doesn't really import, just analyzes

   248 # If it *were* importing, it would be the one-and-only ImportManager

   249 # ie, the builtin import

   250 UNTRIED = -1

   251 

   252 imptyps = ['top-level', 'conditional', 'delayed', 'delayed, conditional']

   253 import hooks

   254 

   255 class ImportTracker:

   256     # really the equivalent of builtin import

   257     def __init__(self, xpath=None, hookspath=None, excludes=None):

   258         self.path = []

   259         self.warnings = {}

   260         if xpath:

   261             self.path = xpath

   262         self.path.extend(sys.path)

   263         self.modules = {}

   264         self.metapath = [

   265             BuiltinImportDirector(),

   266             FrozenImportDirector(),

   267             RegistryImportDirector(),

   268             PathImportDirector(self.path)

   269         ]

   270         if hookspath:

   271             hooks.__path__.extend(hookspath)

   272         self.excludes = excludes

   273         if excludes is None:

   274             self.excludes = []

   275     def analyze_r(self, nm, importernm=None):

   276         importer = importernm

   277         if importer is None:

   278             importer = '__main__'

   279         seen = {}

   280         nms = self.analyze_one(nm, importernm)

   281         nms = map(None, nms, [importer]*len(nms))

   282         i = 0

   283         while i < len(nms):

   284             nm, importer = nms[i]

   285             if seen.get(nm,0):

   286                 del nms[i]

   287                 mod = self.modules[nm]

   288                 if mod:

   289                     mod.xref(importer)

   290             else:

   291                 i = i + 1

   292                 seen[nm] = 1

   293                 j = i

   294                 mod = self.modules[nm]

   295                 if mod:

   296                     mod.xref(importer)

   297                     for name, isdelayed, isconditional in mod.imports:

   298                         imptyp = isdelayed * 2 + isconditional

   299                         newnms = self.analyze_one(name, nm, imptyp)

   300                         newnms = map(None, newnms, [nm]*len(newnms))

   301                         nms[j:j] = newnms

   302                         j = j + len(newnms)

   303         return map(lambda a: a[0], nms)

   304     def analyze_one(self, nm, importernm=None, imptyp=0):

   305         # first see if we could be importing a relative name

   306         contexts = [None]

   307         _all = None

   308         if importernm:

   309             if self.ispackage(importernm):

   310                 contexts.insert(0,importernm)

   311             else:

   312                 pkgnm = string.join(string.split(importernm, '.')[:-1], '.')

   313                 if pkgnm:

   314                     contexts.insert(0,pkgnm)

   315         # so contexts is [pkgnm, None] or just [None]

   316         # now break the name being imported up so we get:

   317         # a.b.c -> [a, b, c]

   318         nmparts = string.split(nm, '.')

   319         if nmparts[-1] == '*':

   320             del nmparts[-1]

   321             _all = []

   322         nms = []

   323         for context in contexts:

   324             ctx = context

   325             for i in range(len(nmparts)):

   326                 nm = nmparts[i]

   327                 if ctx:

   328                     fqname = ctx + '.' + nm

   329                 else:

   330                     fqname = nm

   331                 mod = self.modules.get(fqname, UNTRIED)

   332                 if mod is UNTRIED:

   333                     mod = self.doimport(nm, ctx, fqname)

   334                 if mod:

   335                     nms.append(mod.__name__)

   336                     ctx = fqname

   337                 else:

   338                     break

   339             else:

   340                 # no break, point i beyond end

   341                 i = i + 1

   342             if i:

   343                 break

   344         # now nms is the list of modules that went into sys.modules

   345         # just as result of the structure of the name being imported

   346         # however, each mod has been scanned and that list is in mod.imports

   347         if i<len(nmparts):

   348             if ctx:

   349                 if hasattr(self.modules[ctx], nmparts[i]):

   350                     return nms

   351                 if not self.ispackage(ctx):

   352                     return nms

   353             self.warnings["W: no module named %s (%s import by %s)" % (fqname, imptyps[imptyp], importernm or "__main__")] = 1

   354             if self.modules.has_key(fqname):

   355                 del self.modules[fqname]

   356             return nms

   357         if _all is None:

   358             return nms

   359         bottommod = self.modules[ctx]

   360         if bottommod.ispackage():

   361             for nm in bottommod._all:

   362                 if not hasattr(bottommod, nm):

   363                     mod = self.doimport(nm, ctx, ctx+'.'+nm)

   364                     if mod:

   365                         nms.append(mod.__name__)

   366                     else:

   367                         bottommod.warnings.append("W: name %s not found" % nm)

   368         return nms

   369 

   370     def analyze_script(self, fnm):

   371         try:

   372             co = compile(open(fnm, 'r').read()+'\n', fnm, 'exec')

   373         except SyntaxError, e:

   374             print "Invalid syntax in %s" % fnm

   375             print e.args

   376             raise

   377         mod = PyScript(fnm, co)

   378         self.modules['__main__'] = mod

   379         return self.analyze_r('__main__')

   380 

   381 

   382     def ispackage(self, nm):

   383         return self.modules[nm].ispackage()

   384 

   385     def doimport(self, nm, parentnm, fqname):

   386         # Not that nm is NEVER a dotted name at this point

   387         if fqname in self.excludes:

   388             return None

   389         if parentnm:

   390             parent = self.modules[parentnm]

   391             if parent.ispackage():

   392                 mod = parent.doimport(nm)

   393                 if mod:

   394                     setattr(parent, nm, mod)

   395             else:

   396                 return None

   397         else:

   398             # now we're dealing with an absolute import

   399             for director in self.metapath:

   400                 mod = director.getmod(nm)

   401                 if mod:

   402                     break

   403         if mod:

   404             mod.__name__ = fqname

   405             self.modules[fqname] = mod

   406             # now look for hooks

   407             # this (and scan_code) are instead of doing "exec co in mod.__dict__"

   408             try:

   409                 hookmodnm = 'hook-'+fqname

   410                 hooks = __import__('hooks', globals(), locals(), [hookmodnm])

   411                 hook = getattr(hooks, hookmodnm)

   412                 #print `hook`

   413             except (ImportError, AttributeError):

   414                 pass

   415             else:

   416                 # rearranged so that hook() has a chance to mess with hiddenimports & attrs

   417                 if hasattr(hook, 'hook'):

   418                     mod = hook.hook(mod)

   419                 if hasattr(hook, 'hiddenimports'):

   420                     for impnm in hook.hiddenimports:

   421                         mod.imports.append((impnm, 0, 0))

   422                 if hasattr(hook, 'attrs'):

   423                     for attr, val in hook.attrs:

   424                         setattr(mod, attr, val)

   425 

   426                 if fqname != mod.__name__:

   427                     print "W: %s is changing it's name to %s" % (fqname, mod.__name__)

   428                     self.modules[mod.__name__] = mod

   429         else:

   430             self.modules[fqname] = None

   431         return mod

   432     def getwarnings(self):

   433         warnings = self.warnings.keys()

   434         for nm,mod in self.modules.items():

   435             if mod:

   436                 for w in mod.warnings:

   437                     warnings.append(w+' - %s (%s)' % (mod.__name__, mod.__file__))

   438         return warnings

   439     def getxref(self):

   440         mods = self.modules.items() # (nm, mod)

   441         mods.sort()

   442         rslt = []

   443         for nm, mod in mods:

   444             if mod:

   445                 importers = mod._xref.keys()

   446                 importers.sort()

   447                 rslt.append((nm, importers))

   448         return rslt

   449 

   450 #====================Modules============================#

   451 # All we're doing here is tracking, not importing

   452 # If we were importing, these would be hooked to the real module objects

   453 

   454 class Module:

   455     _ispkg = 0

   456     typ = 'UNKNOWN'

   457     def __init__(self, nm):

   458         self.__name__ = nm

   459         self._all = []

   460         self.imports = []

   461         self.warnings = []

   462         self._xref = {}

   463     def ispackage(self):

   464         return self._ispkg

   465     def doimport(self, nm):

   466         pass

   467     def xref(self, nm):

   468         self._xref[nm] = 1

   469 

   470 class BuiltinModule(Module):

   471     typ = 'BUILTIN'

   472     def __init__(self, nm):

   473         Module.__init__(self, nm)

   474 

   475 class ExtensionModule(Module):

   476     typ = 'EXTENSION'

   477     def __init__(self, nm, pth):

   478         Module.__init__(self, nm)

   479         self.__file__ = pth

   480 

   481 class PyModule(Module):

   482     typ = 'PYMODULE'

   483     def __init__(self, nm, pth, co):

   484         Module.__init__(self, nm)

   485         self.co = co

   486         self.__file__ = pth

   487         if os.path.splitext(self.__file__)[1] == '.py':

   488             if __debug__:

   489                 self.__file__ = self.__file__ + 'c'

   490             else:

   491                 self.__file__ = self.__file__ + 'o'

   492         self.scancode()

   493     def scancode(self):

   494         self.imports, self.warnings, allnms = scan_code(self.co)

   495         if allnms:

   496             self._all = allnms

   497 

   498 class PyScript(PyModule):

   499     typ = 'PYSOURCE'

   500     def __init__(self, pth, co):

   501         Module.__init__(self, '__main__')

   502         self.co = co

   503         self.__file__ = pth

   504         self.scancode()

   505 

   506 class PkgModule(PyModule):

   507     typ = 'PYMODULE'

   508     def __init__(self, nm, pth, co):

   509         PyModule.__init__(self, nm, pth, co)

   510         self._ispkg = 1

   511         pth = os.path.dirname(pth)

   512         self.__path__ = [ pth ]

   513         self.subimporter = PathImportDirector(self.__path__)

   514     def doimport(self, nm):

   515         mod = self.subimporter.getmod(nm)

   516         if mod:

   517             mod.__name__ = self.__name__ + '.' + mod.__name__

   518         return mod

   519 

   520 class PkgInPYZModule(PyModule):

   521     def __init__(self, nm, co, pyzowner):

   522         PyModule.__init__(self, nm, co.co_filename, co)

   523         self._ispkg = 1

   524         self.__path__ = [ str(pyzowner) ]

   525         self.owner = pyzowner

   526     def doimport(self, nm):

   527         mod = self.owner.getmod(self.__name__ + '.' + nm)

   528         return mod

   529 

   530 #======================== Utility ================================#

   531 # Scan the code object for imports, __all__ and wierd stuff

   532 

   533 import dis

   534 IMPORT_NAME = dis.opname.index('IMPORT_NAME')

   535 IMPORT_FROM = dis.opname.index('IMPORT_FROM')

   536 try:

   537     IMPORT_STAR = dis.opname.index('IMPORT_STAR')

   538 except:

   539     IMPORT_STAR = 999

   540 STORE_NAME = dis.opname.index('STORE_NAME')

   541 STORE_FAST = dis.opname.index('STORE_FAST')

   542 STORE_GLOBAL = dis.opname.index('STORE_GLOBAL')

   543 LOAD_GLOBAL = dis.opname.index('LOAD_GLOBAL')

   544 EXEC_STMT = dis.opname.index('EXEC_STMT')

   545 try:

   546     SET_LINENO = dis.opname.index('SET_LINENO')

   547 except ValueError:

   548     SET_LINENO = 999

   549 BUILD_LIST = dis.opname.index('BUILD_LIST')

   550 LOAD_CONST = dis.opname.index('LOAD_CONST')

   551 JUMP_IF_FALSE = dis.opname.index('JUMP_IF_FALSE')

   552 JUMP_IF_TRUE = dis.opname.index('JUMP_IF_TRUE')

   553 JUMP_FORWARD = dis.opname.index('JUMP_FORWARD')

   554 try:

   555     STORE_DEREF = dis.opname.index('STORE_DEREF')

   556 except ValueError:

   557     STORE_DEREF = 999

   558 COND_OPS = [JUMP_IF_TRUE, JUMP_IF_FALSE]

   559 STORE_OPS = [STORE_NAME, STORE_FAST, STORE_GLOBAL, STORE_DEREF]

   560 #IMPORT_STAR -> IMPORT_NAME mod ; IMPORT_STAR

   561 #JUMP_IF_FALSE / JUMP_IF_TRUE / JUMP_FORWARD

   562 

   563 def pass1(code):

   564     instrs = []

   565     i = 0

   566     n = len(code)

   567     curline = 0

   568     incondition = 0

   569     out = 0

   570     while i < n:

   571         if i >= out:

   572             incondition = 0

   573         c = code[i]

   574         i = i+1

   575         op = ord(c)

   576         if op >= dis.HAVE_ARGUMENT:

   577             oparg = ord(code[i]) + ord(code[i+1])*256

   578             i = i+2

   579         else:

   580             oparg = None

   581         if not incondition and op in COND_OPS:

   582             incondition = 1

   583             out = i + oparg

   584         elif incondition and op == JUMP_FORWARD:

   585             out = max(out, i + oparg)

   586         if op == SET_LINENO:

   587             curline = oparg

   588         else:

   589             instrs.append((op, oparg, incondition, curline))

   590     return instrs

   591 

   592 def scan_code(co, m=None, w=None, nested=0):

   593     instrs = pass1(co.co_code)

   594     if m is None:

   595         m = []

   596     if w is None:

   597         w = []

   598     all = None

   599     lastname = None

   600     for i in range(len(instrs)):

   601         op, oparg, conditional, curline = instrs[i]

   602         if op == IMPORT_NAME:

   603             name = lastname = co.co_names[oparg]

   604             m.append((name, nested, conditional))

   605         elif op == IMPORT_FROM:

   606             name = co.co_names[oparg]

   607             m.append((lastname+'.'+name, nested, conditional))

   608             assert lastname is not None

   609         elif op == IMPORT_STAR:

   610             m.append((lastname+'.*', nested, conditional))

   611         elif op == STORE_NAME:

   612             if co.co_names[oparg] == "__all__":

   613                 j = i - 1

   614                 pop, poparg, pcondtl, pline = instrs[j]

   615                 if pop != BUILD_LIST:

   616                     w.append("W: __all__ is built strangely at line %s" % pline)

   617                 else:

   618                     all = []

   619                     while j > 0:

   620                         j = j - 1

   621                         pop, poparg, pcondtl, pline = instrs[j]

   622                         if pop == LOAD_CONST:

   623                             all.append(co.co_consts[poparg])

   624                         else:

   625                             break

   626         elif op in STORE_OPS:

   627             pass

   628         elif op == LOAD_GLOBAL:

   629             name = co.co_names[oparg]

   630             cndtl = ['', 'conditional'][conditional]

   631             lvl = ['top-level', 'delayed'][nested]

   632             if name == "__import__":

   633                 w.append("W: %s %s __import__ hack detected at line %s"  % (lvl, cndtl, curline))

   634             elif name == "eval":

   635                 w.append("W: %s %s eval hack detected at line %s"  % (lvl, cndtl, curline))

   636         elif op == EXEC_STMT:

   637             cndtl = ['', 'conditional'][conditional]

   638             lvl = ['top-level', 'delayed'][nested]

   639             w.append("W: %s %s exec statement detected at line %s"  % (lvl, cndtl, curline))

   640         else:

   641             lastname = None

   642     for c in co.co_consts:

   643         if isinstance(c, type(co)):

   644             scan_code(c, m, w, 1)

   645     return m, w, all