1 """Package Install Manager for Python.

     2 

     3 This is currently a MacOSX-only strawman implementation.

     4 Despite other rumours the name stands for "Packman IMPlementation".

     5 

     6 Tools to allow easy installation of packages. The idea is that there is

     7 an online XML database per (platform, python-version) containing packages

     8 known to work with that combination. This module contains tools for getting

     9 and parsing the database, testing whether packages are installed, computing

    10 dependencies and installing packages.

    11 

    12 There is a minimal main program that works as a command line tool, but the

    13 intention is that the end user will use this through a GUI.

    14 """

    15 

    16 from warnings import warnpy3k

    17 warnpy3k("In 3.x, the pimp module is removed.", stacklevel=2)

    18 

    19 import sys

    20 import os

    21 import subprocess

    22 import urllib

    23 import urllib2

    24 import urlparse

    25 import plistlib

    26 import distutils.util

    27 import distutils.sysconfig

    28 import hashlib

    29 import tarfile

    30 import tempfile

    31 import shutil

    32 import time

    33 

    34 __all__ = ["PimpPreferences", "PimpDatabase", "PimpPackage", "main",

    35     "getDefaultDatabase", "PIMP_VERSION", "main"]

    36 

    37 _scriptExc_NotInstalled = "pimp._scriptExc_NotInstalled"

    38 _scriptExc_OldInstalled = "pimp._scriptExc_OldInstalled"

    39 _scriptExc_BadInstalled = "pimp._scriptExc_BadInstalled"

    40 

    41 NO_EXECUTE=0

    42 

    43 PIMP_VERSION="0.5"

    44 

    45 # Flavors:

    46 # source: setup-based package

    47 # binary: tar (or other) archive created with setup.py bdist.

    48 # installer: something that can be opened

    49 DEFAULT_FLAVORORDER=['source', 'binary', 'installer']

    50 DEFAULT_DOWNLOADDIR='/tmp'

    51 DEFAULT_BUILDDIR='/tmp'

    52 DEFAULT_INSTALLDIR=distutils.sysconfig.get_python_lib()

    53 DEFAULT_PIMPDATABASE_FMT="http://www.python.org/packman/version-%s/%s-%s-%s-%s-%s.plist"

    54 

    55 def getDefaultDatabase(experimental=False):

    56     if experimental:

    57         status = "exp"

    58     else:

    59         status = "prod"

    60 

    61     major, minor, micro, state, extra = sys.version_info

    62     pyvers = '%d.%d' % (major, minor)

    63     if micro == 0 and state != 'final':

    64         pyvers = pyvers + '%s%d' % (state, extra)

    65 

    66     longplatform = distutils.util.get_platform()

    67     osname, release, machine = longplatform.split('-')

    68     # For some platforms we may want to differentiate between

    69     # installation types

    70     if osname == 'darwin':

    71         if sys.prefix.startswith('/System/Library/Frameworks/Python.framework'):

    72             osname = 'darwin_apple'

    73         elif sys.prefix.startswith('/Library/Frameworks/Python.framework'):

    74             osname = 'darwin_macpython'

    75         # Otherwise we don't know...

    76     # Now we try various URLs by playing with the release string.

    77     # We remove numbers off the end until we find a match.

    78     rel = release

    79     while True:

    80         url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, rel, machine)

    81         try:

    82             urllib2.urlopen(url)

    83         except urllib2.HTTPError, arg:

    84             pass

    85         else:

    86             break

    87         if not rel:

    88             # We're out of version numbers to try. Use the

    89             # full release number, this will give a reasonable

    90             # error message later

    91             url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, release, machine)

    92             break

    93         idx = rel.rfind('.')

    94         if idx < 0:

    95             rel = ''

    96         else:

    97             rel = rel[:idx]

    98     return url

    99 

   100 def _cmd(output, dir, *cmditems):

   101     """Internal routine to run a shell command in a given directory."""

   102 

   103     cmd = ("cd \"%s\"; " % dir) + " ".join(cmditems)

   104     if output:

   105         output.write("+ %s\n" % cmd)

   106     if NO_EXECUTE:

   107         return 0

   108     child = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,

   109                              stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

   110     child.stdin.close()

   111     while 1:

   112         line = child.stdout.readline()

   113         if not line:

   114             break

   115         if output:

   116             output.write(line)

   117     return child.wait()

   118 

   119 class PimpDownloader:

   120     """Abstract base class - Downloader for archives"""

   121 

   122     def __init__(self, argument,

   123             dir="",

   124             watcher=None):

   125         self.argument = argument

   126         self._dir = dir

   127         self._watcher = watcher

   128 

   129     def download(self, url, filename, output=None):

   130         return None

   131 

   132     def update(self, str):

   133         if self._watcher:

   134             return self._watcher.update(str)

   135         return True

   136 

   137 class PimpCurlDownloader(PimpDownloader):

   138 

   139     def download(self, url, filename, output=None):

   140         self.update("Downloading %s..." % url)

   141         exitstatus = _cmd(output, self._dir,

   142                     "curl",

   143                     "--output", filename,

   144                     url)

   145         self.update("Downloading %s: finished" % url)

   146         return (not exitstatus)

   147 

   148 class PimpUrllibDownloader(PimpDownloader):

   149 

   150     def download(self, url, filename, output=None):

   151         output = open(filename, 'wb')

   152         self.update("Downloading %s: opening connection" % url)

   153         keepgoing = True

   154         download = urllib2.urlopen(url)

   155         if download.headers.has_key("content-length"):

   156             length = long(download.headers['content-length'])

   157         else:

   158             length = -1

   159 

   160         data = download.read(4096) #read 4K at a time

   161         dlsize = 0

   162         lasttime = 0

   163         while keepgoing:

   164             dlsize = dlsize + len(data)

   165             if len(data) == 0:

   166                 #this is our exit condition

   167                 break

   168             output.write(data)

   169             if int(time.time()) != lasttime:

   170                 # Update at most once per second

   171                 lasttime = int(time.time())

   172                 if length == -1:

   173                     keepgoing = self.update("Downloading %s: %d bytes..." % (url, dlsize))

   174                 else:

   175                     keepgoing = self.update("Downloading %s: %d%% (%d bytes)..." % (url, int(100.0*dlsize/length), dlsize))

   176             data = download.read(4096)

   177         if keepgoing:

   178             self.update("Downloading %s: finished" % url)

   179         return keepgoing

   180 

   181 class PimpUnpacker:

   182     """Abstract base class - Unpacker for archives"""

   183 

   184     _can_rename = False

   185 

   186     def __init__(self, argument,

   187             dir="",

   188             renames=[],

   189             watcher=None):

   190         self.argument = argument

   191         if renames and not self._can_rename:

   192             raise RuntimeError, "This unpacker cannot rename files"

   193         self._dir = dir

   194         self._renames = renames

   195         self._watcher = watcher

   196 

   197     def unpack(self, archive, output=None, package=None):

   198         return None

   199 

   200     def update(self, str):

   201         if self._watcher:

   202             return self._watcher.update(str)

   203         return True

   204 

   205 class PimpCommandUnpacker(PimpUnpacker):

   206     """Unpack archives by calling a Unix utility"""

   207 

   208     _can_rename = False

   209 

   210     def unpack(self, archive, output=None, package=None):

   211         cmd = self.argument % archive

   212         if _cmd(output, self._dir, cmd):

   213             return "unpack command failed"

   214 

   215 class PimpTarUnpacker(PimpUnpacker):

   216     """Unpack tarfiles using the builtin tarfile module"""

   217 

   218     _can_rename = True

   219 

   220     def unpack(self, archive, output=None, package=None):

   221         tf = tarfile.open(archive, "r")

   222         members = tf.getmembers()

   223         skip = []

   224         if self._renames:

   225             for member in members:

   226                 for oldprefix, newprefix in self._renames:

   227                     if oldprefix[:len(self._dir)] == self._dir:

   228                         oldprefix2 = oldprefix[len(self._dir):]

   229                     else:

   230                         oldprefix2 = None

   231                     if member.name[:len(oldprefix)] == oldprefix:

   232                         if newprefix is None:

   233                             skip.append(member)

   234                             #print 'SKIP', member.name

   235                         else:

   236                             member.name = newprefix + member.name[len(oldprefix):]

   237                             print '    ', member.name

   238                         break

   239                     elif oldprefix2 and member.name[:len(oldprefix2)] == oldprefix2:

   240                         if newprefix is None:

   241                             skip.append(member)

   242                             #print 'SKIP', member.name

   243                         else:

   244                             member.name = newprefix + member.name[len(oldprefix2):]

   245                             #print '    ', member.name

   246                         break

   247                 else:

   248                     skip.append(member)

   249                     #print '????', member.name

   250         for member in members:

   251             if member in skip:

   252                 self.update("Skipping %s" % member.name)

   253                 continue

   254             self.update("Extracting %s" % member.name)

   255             tf.extract(member, self._dir)

   256         if skip:

   257             names = [member.name for member in skip if member.name[-1] != '/']

   258             if package:

   259                 names = package.filterExpectedSkips(names)

   260             if names:

   261                 return "Not all files were unpacked: %s" % " ".join(names)

   262 

   263 ARCHIVE_FORMATS = [

   264     (".tar.Z", PimpTarUnpacker, None),

   265     (".taz", PimpTarUnpacker, None),

   266     (".tar.gz", PimpTarUnpacker, None),

   267     (".tgz", PimpTarUnpacker, None),

   268     (".tar.bz", PimpTarUnpacker, None),

   269     (".zip", PimpCommandUnpacker, "unzip \"%s\""),

   270 ]

   271 

   272 class PimpPreferences:

   273     """Container for per-user preferences, such as the database to use

   274     and where to install packages."""

   275 

   276     def __init__(self,

   277             flavorOrder=None,

   278             downloadDir=None,

   279             buildDir=None,

   280             installDir=None,

   281             pimpDatabase=None):

   282         if not flavorOrder:

   283             flavorOrder = DEFAULT_FLAVORORDER

   284         if not downloadDir:

   285             downloadDir = DEFAULT_DOWNLOADDIR

   286         if not buildDir:

   287             buildDir = DEFAULT_BUILDDIR

   288         if not pimpDatabase:

   289             pimpDatabase = getDefaultDatabase()

   290         self.setInstallDir(installDir)

   291         self.flavorOrder = flavorOrder

   292         self.downloadDir = downloadDir

   293         self.buildDir = buildDir

   294         self.pimpDatabase = pimpDatabase

   295         self.watcher = None

   296 

   297     def setWatcher(self, watcher):

   298         self.watcher = watcher

   299 

   300     def setInstallDir(self, installDir=None):

   301         if installDir:

   302             # Installing to non-standard location.

   303             self.installLocations = [

   304                 ('--install-lib', installDir),

   305                 ('--install-headers', None),

   306                 ('--install-scripts', None),

   307                 ('--install-data', None)]

   308         else:

   309             installDir = DEFAULT_INSTALLDIR

   310             self.installLocations = []

   311         self.installDir = installDir

   312 

   313     def isUserInstall(self):

   314         return self.installDir != DEFAULT_INSTALLDIR

   315 

   316     def check(self):

   317         """Check that the preferences make sense: directories exist and are

   318         writable, the install directory is on sys.path, etc."""

   319 

   320         rv = ""

   321         RWX_OK = os.R_OK|os.W_OK|os.X_OK

   322         if not os.path.exists(self.downloadDir):

   323             rv += "Warning: Download directory \"%s\" does not exist\n" % self.downloadDir

   324         elif not os.access(self.downloadDir, RWX_OK):

   325             rv += "Warning: Download directory \"%s\" is not writable or not readable\n" % self.downloadDir

   326         if not os.path.exists(self.buildDir):

   327             rv += "Warning: Build directory \"%s\" does not exist\n" % self.buildDir

   328         elif not os.access(self.buildDir, RWX_OK):

   329             rv += "Warning: Build directory \"%s\" is not writable or not readable\n" % self.buildDir

   330         if not os.path.exists(self.installDir):

   331             rv += "Warning: Install directory \"%s\" does not exist\n" % self.installDir

   332         elif not os.access(self.installDir, RWX_OK):

   333             rv += "Warning: Install directory \"%s\" is not writable or not readable\n" % self.installDir

   334         else:

   335             installDir = os.path.realpath(self.installDir)

   336             for p in sys.path:

   337                 try:

   338                     realpath = os.path.realpath(p)

   339                 except:

   340                     pass

   341                 if installDir == realpath:

   342                     break

   343             else:

   344                 rv += "Warning: Install directory \"%s\" is not on sys.path\n" % self.installDir

   345         return rv

   346 

   347     def compareFlavors(self, left, right):

   348         """Compare two flavor strings. This is part of your preferences

   349         because whether the user prefers installing from source or binary is."""

   350         if left in self.flavorOrder:

   351             if right in self.flavorOrder:

   352                 return cmp(self.flavorOrder.index(left), self.flavorOrder.index(right))

   353             return -1

   354         if right in self.flavorOrder:

   355             return 1

   356         return cmp(left, right)

   357 

   358 class PimpDatabase:

   359     """Class representing a pimp database. It can actually contain

   360     information from multiple databases through inclusion, but the

   361     toplevel database is considered the master, as its maintainer is

   362     "responsible" for the contents."""

   363 

   364     def __init__(self, prefs):

   365         self._packages = []

   366         self.preferences = prefs

   367         self._url = ""

   368         self._urllist = []

   369         self._version = ""

   370         self._maintainer = ""

   371         self._description = ""

   372 

   373     # Accessor functions

   374     def url(self): return self._url

   375     def version(self): return self._version

   376     def maintainer(self): return self._maintainer

   377     def description(self): return self._description

   378 

   379     def close(self):

   380         """Clean up"""

   381         self._packages = []

   382         self.preferences = None

   383 

   384     def appendURL(self, url, included=0):

   385         """Append packages from the database with the given URL.

   386         Only the first database should specify included=0, so the

   387         global information (maintainer, description) get stored."""

   388 

   389         if url in self._urllist:

   390             return

   391         self._urllist.append(url)

   392         fp = urllib2.urlopen(url).fp

   393         plistdata = plistlib.Plist.fromFile(fp)

   394         # Test here for Pimp version, etc

   395         if included:

   396             version = plistdata.get('Version')

   397             if version and version > self._version:

   398                 sys.stderr.write("Warning: included database %s is for pimp version %s\n" %

   399                     (url, version))

   400         else:

   401             self._version = plistdata.get('Version')

   402             if not self._version:

   403                 sys.stderr.write("Warning: database has no Version information\n")

   404             elif self._version > PIMP_VERSION:

   405                 sys.stderr.write("Warning: database version %s newer than pimp version %s\n"

   406                     % (self._version, PIMP_VERSION))

   407             self._maintainer = plistdata.get('Maintainer', '')

   408             self._description = plistdata.get('Description', '').strip()

   409             self._url = url

   410         self._appendPackages(plistdata['Packages'], url)

   411         others = plistdata.get('Include', [])

   412         for o in others:

   413             o = urllib.basejoin(url, o)

   414             self.appendURL(o, included=1)

   415 

   416     def _appendPackages(self, packages, url):

   417         """Given a list of dictionaries containing package

   418         descriptions create the PimpPackage objects and append them

   419         to our internal storage."""

   420 

   421         for p in packages:

   422             p = dict(p)

   423             if p.has_key('Download-URL'):

   424                 p['Download-URL'] = urllib.basejoin(url, p['Download-URL'])

   425             flavor = p.get('Flavor')

   426             if flavor == 'source':

   427                 pkg = PimpPackage_source(self, p)

   428             elif flavor == 'binary':

   429                 pkg = PimpPackage_binary(self, p)

   430             elif flavor == 'installer':

   431                 pkg = PimpPackage_installer(self, p)

   432             elif flavor == 'hidden':

   433                 pkg = PimpPackage_installer(self, p)

   434             else:

   435                 pkg = PimpPackage(self, dict(p))

   436             self._packages.append(pkg)

   437 

   438     def list(self):

   439         """Return a list of all PimpPackage objects in the database."""

   440 

   441         return self._packages

   442 

   443     def listnames(self):

   444         """Return a list of names of all packages in the database."""

   445 

   446         rv = []

   447         for pkg in self._packages:

   448             rv.append(pkg.fullname())

   449         rv.sort()

   450         return rv

   451 

   452     def dump(self, pathOrFile):

   453         """Dump the contents of the database to an XML .plist file.

   454 

   455         The file can be passed as either a file object or a pathname.

   456         All data, including included databases, is dumped."""

   457 

   458         packages = []

   459         for pkg in self._packages:

   460             packages.append(pkg.dump())

   461         plistdata = {

   462             'Version': self._version,

   463             'Maintainer': self._maintainer,

   464             'Description': self._description,

   465             'Packages': packages

   466             }

   467         plist = plistlib.Plist(**plistdata)

   468         plist.write(pathOrFile)

   469 

   470     def find(self, ident):

   471         """Find a package. The package can be specified by name

   472         or as a dictionary with name, version and flavor entries.

   473 

   474         Only name is obligatory. If there are multiple matches the

   475         best one (higher version number, flavors ordered according to

   476         users' preference) is returned."""

   477 

   478         if type(ident) == str:

   479             # Remove ( and ) for pseudo-packages

   480             if ident[0] == '(' and ident[-1] == ')':

   481                 ident = ident[1:-1]

   482             # Split into name-version-flavor

   483             fields = ident.split('-')

   484             if len(fields) < 1 or len(fields) > 3:

   485                 return None

   486             name = fields[0]

   487             if len(fields) > 1:

   488                 version = fields[1]

   489             else:

   490                 version = None

   491             if len(fields) > 2:

   492                 flavor = fields[2]

   493             else:

   494                 flavor = None

   495         else:

   496             name = ident['Name']

   497             version = ident.get('Version')

   498             flavor = ident.get('Flavor')

   499         found = None

   500         for p in self._packages:

   501             if name == p.name() and \

   502                     (not version or version == p.version()) and \

   503                     (not flavor or flavor == p.flavor()):

   504                 if not found or found < p:

   505                     found = p

   506         return found

   507 

   508 ALLOWED_KEYS = [

   509     "Name",

   510     "Version",

   511     "Flavor",

   512     "Description",

   513     "Home-page",

   514     "Download-URL",

   515     "Install-test",

   516     "Install-command",

   517     "Pre-install-command",

   518     "Post-install-command",

   519     "Prerequisites",

   520     "MD5Sum",

   521     "User-install-skips",

   522     "Systemwide-only",

   523 ]

   524 

   525 class PimpPackage:

   526     """Class representing a single package."""

   527 

   528     def __init__(self, db, plistdata):

   529         self._db = db

   530         name = plistdata["Name"]

   531         for k in plistdata.keys():

   532             if not k in ALLOWED_KEYS:

   533                 sys.stderr.write("Warning: %s: unknown key %s\n" % (name, k))

   534         self._dict = plistdata

   535 

   536     def __getitem__(self, key):

   537         return self._dict[key]

   538 

   539     def name(self): return self._dict['Name']

   540     def version(self): return self._dict.get('Version')

   541     def flavor(self): return self._dict.get('Flavor')

   542     def description(self): return self._dict['Description'].strip()

   543     def shortdescription(self): return self.description().splitlines()[0]

   544     def homepage(self): return self._dict.get('Home-page')

   545     def downloadURL(self): return self._dict.get('Download-URL')

   546     def systemwideOnly(self): return self._dict.get('Systemwide-only')

   547 

   548     def fullname(self):

   549         """Return the full name "name-version-flavor" of a package.

   550 

   551         If the package is a pseudo-package, something that cannot be

   552         installed through pimp, return the name in (parentheses)."""

   553 

   554         rv = self._dict['Name']

   555         if self._dict.has_key('Version'):

   556             rv = rv + '-%s' % self._dict['Version']

   557         if self._dict.has_key('Flavor'):

   558             rv = rv + '-%s' % self._dict['Flavor']

   559         if self._dict.get('Flavor') == 'hidden':

   560             # Pseudo-package, show in parentheses

   561             rv = '(%s)' % rv

   562         return rv

   563 

   564     def dump(self):

   565         """Return a dict object containing the information on the package."""

   566         return self._dict

   567 

   568     def __cmp__(self, other):

   569         """Compare two packages, where the "better" package sorts lower."""

   570 

   571         if not isinstance(other, PimpPackage):

   572             return cmp(id(self), id(other))

   573         if self.name() != other.name():

   574             return cmp(self.name(), other.name())

   575         if self.version() != other.version():

   576             return -cmp(self.version(), other.version())

   577         return self._db.preferences.compareFlavors(self.flavor(), other.flavor())

   578 

   579     def installed(self):

   580         """Test wheter the package is installed.

   581 

   582         Returns two values: a status indicator which is one of

   583         "yes", "no", "old" (an older version is installed) or "bad"

   584         (something went wrong during the install test) and a human

   585         readable string which may contain more details."""

   586 

   587         namespace = {

   588             "NotInstalled": _scriptExc_NotInstalled,

   589             "OldInstalled": _scriptExc_OldInstalled,

   590             "BadInstalled": _scriptExc_BadInstalled,

   591             "os": os,

   592             "sys": sys,

   593             }

   594         installTest = self._dict['Install-test'].strip() + '\n'

   595         try:

   596             exec installTest in namespace

   597         except ImportError, arg:

   598             return "no", str(arg)

   599         except _scriptExc_NotInstalled, arg:

   600             return "no", str(arg)

   601         except _scriptExc_OldInstalled, arg:

   602             return "old", str(arg)

   603         except _scriptExc_BadInstalled, arg:

   604             return "bad", str(arg)

   605         except:

   606             sys.stderr.write("-------------------------------------\n")

   607             sys.stderr.write("---- %s: install test got exception\n" % self.fullname())

   608             sys.stderr.write("---- source:\n")

   609             sys.stderr.write(installTest)

   610             sys.stderr.write("---- exception:\n")

   611             import traceback

   612             traceback.print_exc(file=sys.stderr)

   613             if self._db._maintainer:

   614                 sys.stderr.write("---- Please copy this and mail to %s\n" % self._db._maintainer)

   615             sys.stderr.write("-------------------------------------\n")

   616             return "bad", "Package install test got exception"

   617         return "yes", ""

   618 

   619     def prerequisites(self):

   620         """Return a list of prerequisites for this package.

   621 

   622         The list contains 2-tuples, of which the first item is either

   623         a PimpPackage object or None, and the second is a descriptive

   624         string. The first item can be None if this package depends on

   625         something that isn't pimp-installable, in which case the descriptive

   626         string should tell the user what to do."""

   627 

   628         rv = []

   629         if not self._dict.get('Download-URL'):

   630             # For pseudo-packages that are already installed we don't

   631             # return an error message

   632             status, _  = self.installed()

   633             if status == "yes":

   634                 return []

   635             return [(None,

   636                 "Package %s cannot be installed automatically, see the description" %

   637                     self.fullname())]

   638         if self.systemwideOnly() and self._db.preferences.isUserInstall():

   639             return [(None,

   640                 "Package %s can only be installed system-wide" %

   641                     self.fullname())]

   642         if not self._dict.get('Prerequisites'):

   643             return []

   644         for item in self._dict['Prerequisites']:

   645             if type(item) == str:

   646                 pkg = None

   647                 descr = str(item)

   648             else:

   649                 name = item['Name']

   650                 if item.has_key('Version'):

   651                     name = name + '-' + item['Version']

   652                 if item.has_key('Flavor'):

   653                     name = name + '-' + item['Flavor']

   654                 pkg = self._db.find(name)

   655                 if not pkg:

   656                     descr = "Requires unknown %s"%name

   657                 else:

   658                     descr = pkg.shortdescription()

   659             rv.append((pkg, descr))

   660         return rv

   661 

   662 

   663     def downloadPackageOnly(self, output=None):

   664         """Download a single package, if needed.

   665 

   666         An MD5 signature is used to determine whether download is needed,

   667         and to test that we actually downloaded what we expected.

   668         If output is given it is a file-like object that will receive a log

   669         of what happens.

   670 

   671         If anything unforeseen happened the method returns an error message

   672         string.

   673         """

   674 

   675         scheme, loc, path, query, frag = urlparse.urlsplit(self._dict['Download-URL'])

   676         path = urllib.url2pathname(path)

   677         filename = os.path.split(path)[1]

   678         self.archiveFilename = os.path.join(self._db.preferences.downloadDir, filename)

   679         if not self._archiveOK():

   680             if scheme == 'manual':

   681                 return "Please download package manually and save as %s" % self.archiveFilename

   682             downloader = PimpUrllibDownloader(None, self._db.preferences.downloadDir,

   683                 watcher=self._db.preferences.watcher)

   684             if not downloader.download(self._dict['Download-URL'],

   685                     self.archiveFilename, output):

   686                 return "download command failed"

   687         if not os.path.exists(self.archiveFilename) and not NO_EXECUTE:

   688             return "archive not found after download"

   689         if not self._archiveOK():

   690             return "archive does not have correct MD5 checksum"

   691 

   692     def _archiveOK(self):

   693         """Test an archive. It should exist and the MD5 checksum should be correct."""

   694 

   695         if not os.path.exists(self.archiveFilename):

   696             return 0

   697         if not self._dict.get('MD5Sum'):

   698             sys.stderr.write("Warning: no MD5Sum for %s\n" % self.fullname())

   699             return 1

   700         data = open(self.archiveFilename, 'rb').read()

   701         checksum = hashlib.md5(data).hexdigest()

   702         return checksum == self._dict['MD5Sum']

   703 

   704     def unpackPackageOnly(self, output=None):

   705         """Unpack a downloaded package archive."""

   706 

   707         filename = os.path.split(self.archiveFilename)[1]

   708         for ext, unpackerClass, arg in ARCHIVE_FORMATS:

   709             if filename[-len(ext):] == ext:

   710                 break

   711         else:

   712             return "unknown extension for archive file: %s" % filename

   713         self.basename = filename[:-len(ext)]

   714         unpacker = unpackerClass(arg, dir=self._db.preferences.buildDir,

   715                 watcher=self._db.preferences.watcher)

   716         rv = unpacker.unpack(self.archiveFilename, output=output)

   717         if rv:

   718             return rv

   719 

   720     def installPackageOnly(self, output=None):

   721         """Default install method, to be overridden by subclasses"""

   722         return "%s: This package needs to be installed manually (no support for flavor=\"%s\")" \

   723             % (self.fullname(), self._dict.get(flavor, ""))

   724 

   725     def installSinglePackage(self, output=None):

   726         """Download, unpack and install a single package.

   727 

   728         If output is given it should be a file-like object and it

   729         will receive a log of what happened."""

   730 

   731         if not self._dict.get('Download-URL'):

   732             return "%s: This package needs to be installed manually (no Download-URL field)" % self.fullname()

   733         msg = self.downloadPackageOnly(output)

   734         if msg:

   735             return "%s: download: %s" % (self.fullname(), msg)

   736 

   737         msg = self.unpackPackageOnly(output)

   738         if msg:

   739             return "%s: unpack: %s" % (self.fullname(), msg)

   740 

   741         return self.installPackageOnly(output)

   742 

   743     def beforeInstall(self):

   744         """Bookkeeping before installation: remember what we have in site-packages"""

   745         self._old_contents = os.listdir(self._db.preferences.installDir)

   746 

   747     def afterInstall(self):

   748         """Bookkeeping after installation: interpret any new .pth files that have

   749         appeared"""

   750 

   751         new_contents = os.listdir(self._db.preferences.installDir)

   752         for fn in new_contents:

   753             if fn in self._old_contents:

   754                 continue

   755             if fn[-4:] != '.pth':

   756                 continue

   757             fullname = os.path.join(self._db.preferences.installDir, fn)

   758             f = open(fullname)

   759             for line in f.readlines():

   760                 if not line:

   761                     continue

   762                 if line[0] == '#':

   763                     continue

   764                 if line[:6] == 'import':

   765                     exec line

   766                     continue

   767                 if line[-1] == '\n':

   768                     line = line[:-1]

   769                 if not os.path.isabs(line):

   770                     line = os.path.join(self._db.preferences.installDir, line)

   771                 line = os.path.realpath(line)

   772                 if not line in sys.path:

   773                     sys.path.append(line)

   774 

   775     def filterExpectedSkips(self, names):

   776         """Return a list that contains only unpexpected skips"""

   777         if not self._db.preferences.isUserInstall():

   778             return names

   779         expected_skips = self._dict.get('User-install-skips')

   780         if not expected_skips:

   781             return names

   782         newnames = []

   783         for name in names:

   784             for skip in expected_skips:

   785                 if name[:len(skip)] == skip:

   786                     break

   787             else:

   788                 newnames.append(name)

   789         return newnames

   790 

   791 class PimpPackage_binary(PimpPackage):

   792 

   793     def unpackPackageOnly(self, output=None):

   794         """We don't unpack binary packages until installing"""

   795         pass

   796 

   797     def installPackageOnly(self, output=None):

   798         """Install a single source package.

   799 

   800         If output is given it should be a file-like object and it

   801         will receive a log of what happened."""

   802 

   803         if self._dict.has_key('Install-command'):

   804             return "%s: Binary package cannot have Install-command" % self.fullname()

   805 

   806         if self._dict.has_key('Pre-install-command'):

   807             if _cmd(output, '/tmp', self._dict['Pre-install-command']):

   808                 return "pre-install %s: running \"%s\" failed" % \

   809                     (self.fullname(), self._dict['Pre-install-command'])

   810 

   811         self.beforeInstall()

   812 

   813         # Install by unpacking

   814         filename = os.path.split(self.archiveFilename)[1]

   815         for ext, unpackerClass, arg in ARCHIVE_FORMATS:

   816             if filename[-len(ext):] == ext:

   817                 break

   818         else:

   819             return "%s: unknown extension for archive file: %s" % (self.fullname(), filename)

   820         self.basename = filename[:-len(ext)]

   821 

   822         install_renames = []

   823         for k, newloc in self._db.preferences.installLocations:

   824             if not newloc:

   825                 continue

   826             if k == "--install-lib":

   827                 oldloc = DEFAULT_INSTALLDIR

   828             else:

   829                 return "%s: Don't know installLocation %s" % (self.fullname(), k)

   830             install_renames.append((oldloc, newloc))

   831 

   832         unpacker = unpackerClass(arg, dir="/", renames=install_renames)

   833         rv = unpacker.unpack(self.archiveFilename, output=output, package=self)

   834         if rv:

   835             return rv

   836 

   837         self.afterInstall()

   838 

   839         if self._dict.has_key('Post-install-command'):

   840             if _cmd(output, '/tmp', self._dict['Post-install-command']):

   841                 return "%s: post-install: running \"%s\" failed" % \

   842                     (self.fullname(), self._dict['Post-install-command'])

   843 

   844         return None

   845 

   846 

   847 class PimpPackage_source(PimpPackage):

   848 

   849     def unpackPackageOnly(self, output=None):

   850         """Unpack a source package and check that setup.py exists"""

   851         PimpPackage.unpackPackageOnly(self, output)

   852         # Test that a setup script has been create

   853         self._buildDirname = os.path.join(self._db.preferences.buildDir, self.basename)

   854         setupname = os.path.join(self._buildDirname, "setup.py")

   855         if not os.path.exists(setupname) and not NO_EXECUTE:

   856             return "no setup.py found after unpack of archive"

   857 

   858     def installPackageOnly(self, output=None):

   859         """Install a single source package.

   860 

   861         If output is given it should be a file-like object and it

   862         will receive a log of what happened."""

   863 

   864         if self._dict.has_key('Pre-install-command'):

   865             if _cmd(output, self._buildDirname, self._dict['Pre-install-command']):

   866                 return "pre-install %s: running \"%s\" failed" % \

   867                     (self.fullname(), self._dict['Pre-install-command'])

   868 

   869         self.beforeInstall()

   870         installcmd = self._dict.get('Install-command')

   871         if installcmd and self._install_renames:

   872             return "Package has install-command and can only be installed to standard location"

   873         # This is the "bit-bucket" for installations: everything we don't

   874         # want. After installation we check that it is actually empty

   875         unwanted_install_dir = None

   876         if not installcmd:

   877             extra_args = ""

   878             for k, v in self._db.preferences.installLocations:

   879                 if not v:

   880                     # We don't want these files installed. Send them

   881                     # to the bit-bucket.

   882                     if not unwanted_install_dir:

   883                         unwanted_install_dir = tempfile.mkdtemp()

   884                     v = unwanted_install_dir

   885                 extra_args = extra_args + " %s \"%s\"" % (k, v)

   886             installcmd = '"%s" setup.py install %s' % (sys.executable, extra_args)

   887         if _cmd(output, self._buildDirname, installcmd):

   888             return "install %s: running \"%s\" failed" % \

   889                 (self.fullname(), installcmd)

   890         if unwanted_install_dir and os.path.exists(unwanted_install_dir):

   891             unwanted_files = os.listdir(unwanted_install_dir)

   892             if unwanted_files:

   893                 rv = "Warning: some files were not installed: %s" % " ".join(unwanted_files)

   894             else:

   895                 rv = None

   896             shutil.rmtree(unwanted_install_dir)

   897             return rv

   898 

   899         self.afterInstall()

   900 

   901         if self._dict.has_key('Post-install-command'):

   902             if _cmd(output, self._buildDirname, self._dict['Post-install-command']):

   903                 return "post-install %s: running \"%s\" failed" % \

   904                     (self.fullname(), self._dict['Post-install-command'])

   905         return None

   906 

   907 class PimpPackage_installer(PimpPackage):

   908 

   909     def unpackPackageOnly(self, output=None):

   910         """We don't unpack dmg packages until installing"""

   911         pass

   912 

   913     def installPackageOnly(self, output=None):

   914         """Install a single source package.

   915 

   916         If output is given it should be a file-like object and it

   917         will receive a log of what happened."""

   918 

   919         if self._dict.has_key('Post-install-command'):

   920             return "%s: Installer package cannot have Post-install-command" % self.fullname()

   921 

   922         if self._dict.has_key('Pre-install-command'):

   923             if _cmd(output, '/tmp', self._dict['Pre-install-command']):

   924                 return "pre-install %s: running \"%s\" failed" % \

   925                     (self.fullname(), self._dict['Pre-install-command'])

   926 

   927         self.beforeInstall()

   928 

   929         installcmd = self._dict.get('Install-command')

   930         if installcmd:

   931             if '%' in installcmd:

   932                 installcmd = installcmd % self.archiveFilename

   933         else:

   934             installcmd = 'open \"%s\"' % self.archiveFilename

   935         if _cmd(output, "/tmp", installcmd):

   936             return '%s: install command failed (use verbose for details)' % self.fullname()

   937         return '%s: downloaded and opened. Install manually and restart Package Manager' % self.archiveFilename

   938 

   939 class PimpInstaller:

   940     """Installer engine: computes dependencies and installs

   941     packages in the right order."""

   942 

   943     def __init__(self, db):

   944         self._todo = []

   945         self._db = db

   946         self._curtodo = []

   947         self._curmessages = []

   948 

   949     def __contains__(self, package):

   950         return package in self._todo

   951 

   952     def _addPackages(self, packages):

   953         for package in packages:

   954             if not package in self._todo:

   955                 self._todo.append(package)

   956 

   957     def _prepareInstall(self, package, force=0, recursive=1):

   958         """Internal routine, recursive engine for prepareInstall.

   959 

   960         Test whether the package is installed and (if not installed

   961         or if force==1) prepend it to the temporary todo list and

   962         call ourselves recursively on all prerequisites."""

   963 

   964         if not force:

   965             status, message = package.installed()

   966             if status == "yes":

   967                 return

   968         if package in self._todo or package in self._curtodo:

   969             return

   970         self._curtodo.insert(0, package)

   971         if not recursive:

   972             return

   973         prereqs = package.prerequisites()

   974         for pkg, descr in prereqs:

   975             if pkg:

   976                 self._prepareInstall(pkg, False, recursive)

   977             else:

   978                 self._curmessages.append("Problem with dependency: %s" % descr)

   979 

   980     def prepareInstall(self, package, force=0, recursive=1):

   981         """Prepare installation of a package.

   982 

   983         If the package is already installed and force is false nothing

   984         is done. If recursive is true prerequisites are installed first.

   985 

   986         Returns a list of packages (to be passed to install) and a list

   987         of messages of any problems encountered.

   988         """

   989 

   990         self._curtodo = []

   991         self._curmessages = []

   992         self._prepareInstall(package, force, recursive)

   993         rv = self._curtodo, self._curmessages

   994         self._curtodo = []

   995         self._curmessages = []

   996         return rv

   997 

   998     def install(self, packages, output):

   999         """Install a list of packages."""

  1000 

  1001         self._addPackages(packages)

  1002         status = []

  1003         for pkg in self._todo:

  1004             msg = pkg.installSinglePackage(output)

  1005             if msg:

  1006                 status.append(msg)

  1007         return status

  1008 

  1009 

  1010 

  1011 def _run(mode, verbose, force, args, prefargs, watcher):

  1012     """Engine for the main program"""

  1013 

  1014     prefs = PimpPreferences(**prefargs)

  1015     if watcher:

  1016         prefs.setWatcher(watcher)

  1017     rv = prefs.check()

  1018     if rv:

  1019         sys.stdout.write(rv)

  1020     db = PimpDatabase(prefs)

  1021     db.appendURL(prefs.pimpDatabase)

  1022 

  1023     if mode == 'dump':

  1024         db.dump(sys.stdout)

  1025     elif mode =='list':

  1026         if not args:

  1027             args = db.listnames()

  1028         print "%-20.20s\t%s" % ("Package", "Description")

  1029         print

  1030         for pkgname in args:

  1031             pkg = db.find(pkgname)

  1032             if pkg:

  1033                 description = pkg.shortdescription()

  1034                 pkgname = pkg.fullname()

  1035             else:

  1036                 description = 'Error: no such package'

  1037             print "%-20.20s\t%s" % (pkgname, description)

  1038             if verbose:

  1039                 print "\tHome page:\t", pkg.homepage()

  1040                 try:

  1041                     print "\tDownload URL:\t", pkg.downloadURL()

  1042                 except KeyError:

  1043                     pass

  1044                 description = pkg.description()

  1045                 description = '\n\t\t\t\t\t'.join(description.splitlines())

  1046                 print "\tDescription:\t%s" % description

  1047     elif mode =='status':

  1048         if not args:

  1049             args = db.listnames()

  1050             print "%-20.20s\t%s\t%s" % ("Package", "Installed", "Message")

  1051             print

  1052         for pkgname in args:

  1053             pkg = db.find(pkgname)

  1054             if pkg:

  1055                 status, msg = pkg.installed()

  1056                 pkgname = pkg.fullname()

  1057             else:

  1058                 status = 'error'

  1059                 msg = 'No such package'

  1060             print "%-20.20s\t%-9.9s\t%s" % (pkgname, status, msg)

  1061             if verbose and status == "no":

  1062                 prereq = pkg.prerequisites()

  1063                 for pkg, msg in prereq:

  1064                     if not pkg:

  1065                         pkg = ''

  1066                     else:

  1067                         pkg = pkg.fullname()

  1068                     print "%-20.20s\tRequirement: %s %s" % ("", pkg, msg)

  1069     elif mode == 'install':

  1070         if not args:

  1071             print 'Please specify packages to install'

  1072             sys.exit(1)

  1073         inst = PimpInstaller(db)

  1074         for pkgname in args:

  1075             pkg = db.find(pkgname)

  1076             if not pkg:

  1077                 print '%s: No such package' % pkgname

  1078                 continue

  1079             list, messages = inst.prepareInstall(pkg, force)

  1080             if messages and not force:

  1081                 print "%s: Not installed:" % pkgname

  1082                 for m in messages:

  1083                     print "\t", m

  1084             else:

  1085                 if verbose:

  1086                     output = sys.stdout

  1087                 else:

  1088                     output = None

  1089                 messages = inst.install(list, output)

  1090                 if messages:

  1091                     print "%s: Not installed:" % pkgname

  1092                     for m in messages:

  1093                         print "\t", m

  1094 

  1095 def main():

  1096     """Minimal commandline tool to drive pimp."""

  1097 

  1098     import getopt

  1099     def _help():

  1100         print "Usage: pimp [options] -s [package ...]  List installed status"

  1101         print "       pimp [options] -l [package ...]  Show package information"

  1102         print "       pimp [options] -i package ...    Install packages"

  1103         print "       pimp -d                          Dump database to stdout"

  1104         print "       pimp -V                          Print version number"

  1105         print "Options:"

  1106         print "       -v     Verbose"

  1107         print "       -f     Force installation"

  1108         print "       -D dir Set destination directory"

  1109         print "              (default: %s)" % DEFAULT_INSTALLDIR

  1110         print "       -u url URL for database"

  1111         sys.exit(1)

  1112 

  1113     class _Watcher:

  1114         def update(self, msg):

  1115             sys.stderr.write(msg + '\r')

  1116             return 1

  1117 

  1118     try:

  1119         opts, args = getopt.getopt(sys.argv[1:], "slifvdD:Vu:")

  1120     except getopt.GetoptError:

  1121         _help()

  1122     if not opts and not args:

  1123         _help()

  1124     mode = None

  1125     force = 0

  1126     verbose = 0

  1127     prefargs = {}

  1128     watcher = None

  1129     for o, a in opts:

  1130         if o == '-s':

  1131             if mode:

  1132                 _help()

  1133             mode = 'status'

  1134         if o == '-l':

  1135             if mode: