1 """distutils.util

     2 

     3 Miscellaneous utility functions -- anything that doesn't fit into

     4 one of the other *util.py modules.

     5 """

     6 

     7 __revision__ = "$Id: util.py 63955 2008-06-05 12:58:24Z ronald.oussoren $"

     8 

     9 import sys, os, string, re

    10 from distutils.errors import DistutilsPlatformError

    11 from distutils.dep_util import newer

    12 from distutils.spawn import spawn

    13 from distutils import log

    14 

    15 def get_platform ():

    16     """Return a string that identifies the current platform.  This is used

    17     mainly to distinguish platform-specific build directories and

    18     platform-specific built distributions.  Typically includes the OS name

    19     and version and the architecture (as supplied by 'os.uname()'),

    20     although the exact information included depends on the OS; eg. for IRIX

    21     the architecture isn't particularly important (IRIX only runs on SGI

    22     hardware), but for Linux the kernel version isn't particularly

    23     important.

    24 

    25     Examples of returned values:

    26        linux-i586

    27        linux-alpha (?)

    28        solaris-2.6-sun4u

    29        irix-5.3

    30        irix64-6.2

    31 

    32     Windows will return one of:

    33        win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)

    34        win-ia64 (64bit Windows on Itanium)

    35        win32 (all others - specifically, sys.platform is returned)

    36 

    37     For other non-POSIX platforms, currently just returns 'sys.platform'.

    38     """

    39     if os.name == 'nt':

    40         # sniff sys.version for architecture.

    41         prefix = " bit ("

    42         i = string.find(sys.version, prefix)

    43         if i == -1:

    44             return sys.platform

    45         j = string.find(sys.version, ")", i)

    46         look = sys.version[i+len(prefix):j].lower()

    47         if look=='amd64':

    48             return 'win-amd64'

    49         if look=='itanium':

    50             return 'win-ia64'

    51         return sys.platform

    52 

    53     if os.name != "posix" or not hasattr(os, 'uname'):

    54         # XXX what about the architecture? NT is Intel or Alpha,

    55         # Mac OS is M68k or PPC, etc.

    56         return sys.platform

    57 

    58     # Try to distinguish various flavours of Unix

    59 

    60     (osname, host, release, version, machine) = os.uname()

    61 

    62     # Convert the OS name to lowercase, remove '/' characters

    63     # (to accommodate BSD/OS), and translate spaces (for "Power Macintosh")

    64     osname = string.lower(osname)

    65     osname = string.replace(osname, '/', '')

    66     machine = string.replace(machine, ' ', '_')

    67     machine = string.replace(machine, '/', '-')

    68 

    69     if osname[:5] == "linux":

    70         # At least on Linux/Intel, 'machine' is the processor --

    71         # i386, etc.

    72         # XXX what about Alpha, SPARC, etc?

    73         return  "%s-%s" % (osname, machine)

    74     elif osname[:5] == "sunos":

    75         if release[0] >= "5":           # SunOS 5 == Solaris 2

    76             osname = "solaris"

    77             release = "%d.%s" % (int(release[0]) - 3, release[2:])

    78         # fall through to standard osname-release-machine representation

    79     elif osname[:4] == "irix":              # could be "irix64"!

    80         return "%s-%s" % (osname, release)

    81     elif osname[:3] == "aix":

    82         return "%s-%s.%s" % (osname, version, release)

    83     elif osname[:6] == "cygwin":

    84         osname = "cygwin"

    85         rel_re = re.compile (r'[\d.]+')

    86         m = rel_re.match(release)

    87         if m:

    88             release = m.group()

    89     elif osname[:6] == "darwin":

    90         #

    91         # For our purposes, we'll assume that the system version from

    92         # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set

    93         # to. This makes the compatibility story a bit more sane because the

    94         # machine is going to compile and link as if it were

    95         # MACOSX_DEPLOYMENT_TARGET.

    96         from distutils.sysconfig import get_config_vars

    97         cfgvars = get_config_vars()

    98 

    99         macver = os.environ.get('MACOSX_DEPLOYMENT_TARGET')

   100         if not macver:

   101             macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')

   102 

   103         if not macver:

   104             # Get the system version. Reading this plist is a documented

   105             # way to get the system version (see the documentation for

   106             # the Gestalt Manager)

   107             try:

   108                 f = open('/System/Library/CoreServices/SystemVersion.plist')

   109             except IOError:

   110                 # We're on a plain darwin box, fall back to the default

   111                 # behaviour.

   112                 pass

   113             else:

   114                 m = re.search(

   115                         r'<key>ProductUserVisibleVersion</key>\s*' +

   116                         r'<string>(.*?)</string>', f.read())

   117                 f.close()

   118                 if m is not None:

   119                     macver = '.'.join(m.group(1).split('.')[:2])

   120                 # else: fall back to the default behaviour

   121 

   122         if macver:

   123             from distutils.sysconfig import get_config_vars

   124             release = macver

   125             osname = "macosx"

   126 

   127 

   128             if (release + '.') >= '10.4.' and \

   129                     '-arch' in get_config_vars().get('CFLAGS', '').strip():

   130                 # The universal build will build fat binaries, but not on

   131                 # systems before 10.4

   132                 #

   133                 # Try to detect 4-way universal builds, those have machine-type

   134                 # 'universal' instead of 'fat'.

   135 

   136                 machine = 'fat'

   137 

   138                 if '-arch x86_64' in get_config_vars().get('CFLAGS'):

   139                     machine = 'universal'

   140 

   141             elif machine in ('PowerPC', 'Power_Macintosh'):

   142                 # Pick a sane name for the PPC architecture.

   143                 machine = 'ppc'

   144 

   145     return "%s-%s-%s" % (osname, release, machine)

   146 

   147 # get_platform ()

   148 

   149 

   150 def convert_path (pathname):

   151     """Return 'pathname' as a name that will work on the native filesystem,

   152     i.e. split it on '/' and put it back together again using the current

   153     directory separator.  Needed because filenames in the setup script are

   154     always supplied in Unix style, and have to be converted to the local

   155     convention before we can actually use them in the filesystem.  Raises

   156     ValueError on non-Unix-ish systems if 'pathname' either starts or

   157     ends with a slash.

   158     """

   159     if os.sep == '/':

   160         return pathname

   161     if not pathname:

   162         return pathname

   163     if pathname[0] == '/':

   164         raise ValueError, "path '%s' cannot be absolute" % pathname

   165     if pathname[-1] == '/':

   166         raise ValueError, "path '%s' cannot end with '/'" % pathname

   167 

   168     paths = string.split(pathname, '/')

   169     while '.' in paths:

   170         paths.remove('.')

   171     if not paths:

   172         return os.curdir

   173     return apply(os.path.join, paths)

   174 

   175 # convert_path ()

   176 

   177 

   178 def change_root (new_root, pathname):

   179     """Return 'pathname' with 'new_root' prepended.  If 'pathname' is

   180     relative, this is equivalent to "os.path.join(new_root,pathname)".

   181     Otherwise, it requires making 'pathname' relative and then joining the

   182     two, which is tricky on DOS/Windows and Mac OS.

   183     """

   184     if os.name == 'posix':

   185         if not os.path.isabs(pathname):

   186             return os.path.join(new_root, pathname)

   187         else:

   188             return os.path.join(new_root, pathname[1:])

   189 

   190     elif os.name == 'nt':

   191         (drive, path) = os.path.splitdrive(pathname)

   192         if path[0] == '\\':

   193             path = path[1:]

   194         return os.path.join(new_root, path)

   195 

   196     elif os.name == 'os2':

   197         (drive, path) = os.path.splitdrive(pathname)

   198         if path[0] == os.sep:

   199             path = path[1:]

   200         return os.path.join(new_root, path)

   201 

   202     elif os.name == 'mac':

   203         if not os.path.isabs(pathname):

   204             return os.path.join(new_root, pathname)

   205         else:

   206             # Chop off volume name from start of path

   207             elements = string.split(pathname, ":", 1)

   208             pathname = ":" + elements[1]

   209             return os.path.join(new_root, pathname)

   210 

   211     else:

   212         raise DistutilsPlatformError, \

   213               "nothing known about platform '%s'" % os.name

   214 

   215 

   216 _environ_checked = 0

   217 def check_environ ():

   218     """Ensure that 'os.environ' has all the environment variables we

   219     guarantee that users can use in config files, command-line options,

   220     etc.  Currently this includes:

   221       HOME - user's home directory (Unix only)

   222       PLAT - description of the current platform, including hardware

   223              and OS (see 'get_platform()')

   224     """

   225     global _environ_checked

   226     if _environ_checked:

   227         return

   228 

   229     if os.name == 'posix' and 'HOME' not in os.environ:

   230         import pwd

   231         os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]

   232 

   233     if 'PLAT' not in os.environ:

   234         os.environ['PLAT'] = get_platform()

   235 

   236     _environ_checked = 1

   237 

   238 

   239 def subst_vars (s, local_vars):

   240     """Perform shell/Perl-style variable substitution on 'string'.  Every

   241     occurrence of '$' followed by a name is considered a variable, and

   242     variable is substituted by the value found in the 'local_vars'

   243     dictionary, or in 'os.environ' if it's not in 'local_vars'.

   244     'os.environ' is first checked/augmented to guarantee that it contains

   245     certain values: see 'check_environ()'.  Raise ValueError for any

   246     variables not found in either 'local_vars' or 'os.environ'.

   247     """

   248     check_environ()

   249     def _subst (match, local_vars=local_vars):

   250         var_name = match.group(1)

   251         if var_name in local_vars:

   252             return str(local_vars[var_name])

   253         else:

   254             return os.environ[var_name]

   255 

   256     try:

   257         return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)

   258     except KeyError, var:

   259         raise ValueError, "invalid variable '$%s'" % var

   260 

   261 # subst_vars ()

   262 

   263 

   264 def grok_environment_error (exc, prefix="error: "):

   265     """Generate a useful error message from an EnvironmentError (IOError or

   266     OSError) exception object.  Handles Python 1.5.1 and 1.5.2 styles, and

   267     does what it can to deal with exception objects that don't have a

   268     filename (which happens when the error is due to a two-file operation,

   269     such as 'rename()' or 'link()'.  Returns the error message as a string

   270     prefixed with 'prefix'.

   271     """

   272     # check for Python 1.5.2-style {IO,OS}Error exception objects

   273     if hasattr(exc, 'filename') and hasattr(exc, 'strerror'):

   274         if exc.filename:

   275             error = prefix + "%s: %s" % (exc.filename, exc.strerror)

   276         else:

   277             # two-argument functions in posix module don't

   278             # include the filename in the exception object!

   279             error = prefix + "%s" % exc.strerror

   280     else:

   281         error = prefix + str(exc[-1])

   282 

   283     return error

   284 

   285 

   286 # Needed by 'split_quoted()'

   287 _wordchars_re = _squote_re = _dquote_re = None

   288 def _init_regex():

   289     global _wordchars_re, _squote_re, _dquote_re

   290     _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)

   291     _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")

   292     _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')

   293 

   294 def split_quoted (s):

   295     """Split a string up according to Unix shell-like rules for quotes and

   296     backslashes.  In short: words are delimited by spaces, as long as those

   297     spaces are not escaped by a backslash, or inside a quoted string.

   298     Single and double quotes are equivalent, and the quote characters can

   299     be backslash-escaped.  The backslash is stripped from any two-character

   300     escape sequence, leaving only the escaped character.  The quote

   301     characters are stripped from any quoted string.  Returns a list of

   302     words.

   303     """

   304 

   305     # This is a nice algorithm for splitting up a single string, since it

   306     # doesn't require character-by-character examination.  It was a little

   307     # bit of a brain-bender to get it working right, though...

   308     if _wordchars_re is None: _init_regex()

   309 

   310     s = string.strip(s)

   311     words = []

   312     pos = 0

   313 

   314     while s:

   315         m = _wordchars_re.match(s, pos)

   316         end = m.end()

   317         if end == len(s):

   318             words.append(s[:end])

   319             break

   320 

   321         if s[end] in string.whitespace: # unescaped, unquoted whitespace: now

   322             words.append(s[:end])       # we definitely have a word delimiter

   323             s = string.lstrip(s[end:])

   324             pos = 0

   325 

   326         elif s[end] == '\\':            # preserve whatever is being escaped;

   327                                         # will become part of the current word

   328             s = s[:end] + s[end+1:]

   329             pos = end+1

   330 

   331         else:

   332             if s[end] == "'":           # slurp singly-quoted string

   333                 m = _squote_re.match(s, end)

   334             elif s[end] == '"':         # slurp doubly-quoted string

   335                 m = _dquote_re.match(s, end)

   336             else:

   337                 raise RuntimeError, \

   338                       "this can't happen (bad char '%c')" % s[end]

   339 

   340             if m is None:

   341                 raise ValueError, \

   342                       "bad string (mismatched %s quotes?)" % s[end]

   343 

   344             (beg, end) = m.span()

   345             s = s[:beg] + s[beg+1:end-1] + s[end:]

   346             pos = m.end() - 2

   347 

   348         if pos >= len(s):

   349             words.append(s)

   350             break

   351 

   352     return words

   353 

   354 # split_quoted ()

   355 

   356 

   357 def execute (func, args, msg=None, verbose=0, dry_run=0):

   358     """Perform some action that affects the outside world (eg.  by

   359     writing to the filesystem).  Such actions are special because they

   360     are disabled by the 'dry_run' flag.  This method takes care of all

   361     that bureaucracy for you; all you have to do is supply the

   362     function to call and an argument tuple for it (to embody the

   363     "external action" being performed), and an optional message to

   364     print.

   365     """

   366     if msg is None:

   367         msg = "%s%r" % (func.__name__, args)

   368         if msg[-2:] == ',)':        # correct for singleton tuple

   369             msg = msg[0:-2] + ')'

   370 

   371     log.info(msg)

   372     if not dry_run:

   373         apply(func, args)

   374 

   375 

   376 def strtobool (val):

   377     """Convert a string representation of truth to true (1) or false (0).

   378 

   379     True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values

   380     are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if

   381     'val' is anything else.

   382     """

   383     val = string.lower(val)

   384     if val in ('y', 'yes', 't', 'true', 'on', '1'):

   385         return 1

   386     elif val in ('n', 'no', 'f', 'false', 'off', '0'):

   387         return 0

   388     else:

   389         raise ValueError, "invalid truth value %r" % (val,)

   390 

   391 

   392 def byte_compile (py_files,

   393                   optimize=0, force=0,

   394                   prefix=None, base_dir=None,

   395                   verbose=1, dry_run=0,

   396                   direct=None):

   397     """Byte-compile a collection of Python source files to either .pyc

   398     or .pyo files in the same directory.  'py_files' is a list of files

   399     to compile; any files that don't end in ".py" are silently skipped.

   400     'optimize' must be one of the following:

   401       0 - don't optimize (generate .pyc)

   402       1 - normal optimization (like "python -O")

   403       2 - extra optimization (like "python -OO")

   404     If 'force' is true, all files are recompiled regardless of

   405     timestamps.

   406 

   407     The source filename encoded in each bytecode file defaults to the

   408     filenames listed in 'py_files'; you can modify these with 'prefix' and

   409     'basedir'.  'prefix' is a string that will be stripped off of each

   410     source filename, and 'base_dir' is a directory name that will be

   411     prepended (after 'prefix' is stripped).  You can supply either or both

   412     (or neither) of 'prefix' and 'base_dir', as you wish.

   413 

   414     If 'dry_run' is true, doesn't actually do anything that would

   415     affect the filesystem.

   416 

   417     Byte-compilation is either done directly in this interpreter process

   418     with the standard py_compile module, or indirectly by writing a

   419     temporary script and executing it.  Normally, you should let

   420     'byte_compile()' figure out to use direct compilation or not (see

   421     the source for details).  The 'direct' flag is used by the script

   422     generated in indirect mode; unless you know what you're doing, leave

   423     it set to None.

   424     """

   425 

   426     # First, if the caller didn't force us into direct or indirect mode,

   427     # figure out which mode we should be in.  We take a conservative

   428     # approach: choose direct mode *only* if the current interpreter is

   429     # in debug mode and optimize is 0.  If we're not in debug mode (-O

   430     # or -OO), we don't know which level of optimization this

   431     # interpreter is running with, so we can't do direct

   432     # byte-compilation and be certain that it's the right thing.  Thus,

   433     # always compile indirectly if the current interpreter is in either

   434     # optimize mode, or if either optimization level was requested by

   435     # the caller.

   436     if direct is None:

   437         direct = (__debug__ and optimize == 0)

   438 

   439     # "Indirect" byte-compilation: write a temporary script and then

   440     # run it with the appropriate flags.

   441     if not direct:

   442         try:

   443             from tempfile import mkstemp

   444             (script_fd, script_name) = mkstemp(".py")

   445         except ImportError:

   446             from tempfile import mktemp

   447             (script_fd, script_name) = None, mktemp(".py")

   448         log.info("writing byte-compilation script '%s'", script_name)

   449         if not dry_run:

   450             if script_fd is not None:

   451                 script = os.fdopen(script_fd, "w")

   452             else:

   453                 script = open(script_name, "w")

   454 

   455             script.write("""\

   456 from distutils.util import byte_compile

   457 files = [

   458 """)

   459 

   460             # XXX would be nice to write absolute filenames, just for

   461             # safety's sake (script should be more robust in the face of

   462             # chdir'ing before running it).  But this requires abspath'ing

   463             # 'prefix' as well, and that breaks the hack in build_lib's

   464             # 'byte_compile()' method that carefully tacks on a trailing

   465             # slash (os.sep really) to make sure the prefix here is "just

   466             # right".  This whole prefix business is rather delicate -- the

   467             # problem is that it's really a directory, but I'm treating it

   468             # as a dumb string, so trailing slashes and so forth matter.

   469 

   470             #py_files = map(os.path.abspath, py_files)

   471             #if prefix:

   472             #    prefix = os.path.abspath(prefix)

   473 

   474             script.write(string.join(map(repr, py_files), ",\n") + "]\n")

   475             script.write("""

   476 byte_compile(files, optimize=%r, force=%r,

   477              prefix=%r, base_dir=%r,

   478              verbose=%r, dry_run=0,

   479              direct=1)

   480 """ % (optimize, force, prefix, base_dir, verbose))

   481 

   482             script.close()

   483 

   484         cmd = [sys.executable, script_name]

   485         if optimize == 1:

   486             cmd.insert(1, "-O")

   487         elif optimize == 2:

   488             cmd.insert(1, "-OO")

   489         spawn(cmd, dry_run=dry_run)

   490         execute(os.remove, (script_name,), "removing %s" % script_name,

   491                 dry_run=dry_run)

   492 

   493     # "Direct" byte-compilation: use the py_compile module to compile

   494     # right here, right now.  Note that the script generated in indirect

   495     # mode simply calls 'byte_compile()' in direct mode, a weird sort of

   496     # cross-process recursion.  Hey, it works!

   497     else:

   498         from py_compile import compile

   499 

   500         for file in py_files:

   501             if file[-3:] != ".py":

   502                 # This lets us be lazy and not filter filenames in

   503                 # the "install_lib" command.

   504                 continue

   505 

   506             # Terminology from the py_compile module:

   507             #   cfile - byte-compiled file

   508             #   dfile - purported source filename (same as 'file' by default)

   509             cfile = file + (__debug__ and "c" or "o")

   510             dfile = file

   511             if prefix:

   512                 if file[:len(prefix)] != prefix:

   513                     raise ValueError, \

   514                           ("invalid prefix: filename %r doesn't start with %r"

   515                            % (file, prefix))

   516                 dfile = dfile[len(prefix):]

   517             if base_dir:

   518                 dfile = os.path.join(base_dir, dfile)

   519 

   520             cfile_base = os.path.basename(cfile)

   521             if direct:

   522                 if force or newer(file, cfile):

   523                     log.info("byte-compiling %s to %s", file, cfile_base)

   524                     if not dry_run:

   525                         compile(file, cfile, dfile)

   526                 else:

   527                     log.debug("skipping byte-compilation of %s to %s",

   528                               file, cfile_base)

   529 

   530 # byte_compile ()

   531 

   532 def rfc822_escape (header):

   533     """Return a version of the string escaped for inclusion in an

   534     RFC-822 header, by ensuring there are 8 spaces space after each newline.

   535     """

   536     lines = string.split(header, '\n')

   537     lines = map(string.strip, lines)

   538     header = string.join(lines, '\n' + 8*' ')

   539     return header