1 """Open an arbitrary URL.

     2 

     3 See the following document for more info on URLs:

     4 "Names and Addresses, URIs, URLs, URNs, URCs", at

     5 http://www.w3.org/pub/WWW/Addressing/Overview.html

     6 

     7 See also the HTTP spec (from which the error codes are derived):

     8 "HTTP - Hypertext Transfer Protocol", at

     9 http://www.w3.org/pub/WWW/Protocols/

    10 

    11 Related standards and specs:

    12 - RFC1808: the "relative URL" spec. (authoritative status)

    13 - RFC1738 - the "URL standard". (authoritative status)

    14 - RFC1630 - the "URI spec". (informational status)

    15 

    16 The object returned by URLopener().open(file) will differ per

    17 protocol.  All you know is that is has methods read(), readline(),

    18 readlines(), fileno(), close() and info().  The read*(), fileno()

    19 and close() methods work like those of open files.

    20 The info() method returns a mimetools.Message object which can be

    21 used to query various info about the object, if available.

    22 (mimetools.Message objects are queried with the getheader() method.)

    23 """

    24 

    25 import string

    26 import socket

    27 import os

    28 import time

    29 import sys

    30 from urlparse import urljoin as basejoin

    31 import warnings

    32 

    33 __all__ = ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",

    34            "urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",

    35            "urlencode", "url2pathname", "pathname2url", "splittag",

    36            "localhost", "thishost", "ftperrors", "basejoin", "unwrap",

    37            "splittype", "splithost", "splituser", "splitpasswd", "splitport",

    38            "splitnport", "splitquery", "splitattr", "splitvalue",

    39            "getproxies"]

    40 

    41 __version__ = '1.17'    # XXX This version is not always updated :-(

    42 

    43 MAXFTPCACHE = 10        # Trim the ftp cache beyond this size

    44 

    45 # Helper for non-unix systems

    46 if os.name == 'mac':

    47     from macurl2path import url2pathname, pathname2url

    48 elif os.name == 'nt':

    49     from nturl2path import url2pathname, pathname2url

    50 elif os.name == 'riscos':

    51     from rourl2path import url2pathname, pathname2url

    52 else:

    53     def url2pathname(pathname):

    54         """OS-specific conversion from a relative URL of the 'file' scheme

    55         to a file system path; not recommended for general use."""

    56         return unquote(pathname)

    57 

    58     def pathname2url(pathname):

    59         """OS-specific conversion from a file system path to a relative URL

    60         of the 'file' scheme; not recommended for general use."""

    61         return quote(pathname)

    62 

    63 # This really consists of two pieces:

    64 # (1) a class which handles opening of all sorts of URLs

    65 #     (plus assorted utilities etc.)

    66 # (2) a set of functions for parsing URLs

    67 # XXX Should these be separated out into different modules?

    68 

    69 

    70 # Shortcut for basic usage

    71 _urlopener = None

    72 def urlopen(url, data=None, proxies=None):

    73     """Create a file-like object for the specified URL to read from."""

    74     from warnings import warnpy3k

    75     warnings.warnpy3k("urllib.urlopen() has been removed in Python 3.0 in "

    76                         "favor of urllib2.urlopen()", stacklevel=2)

    77 

    78     global _urlopener

    79     if proxies is not None:

    80         opener = FancyURLopener(proxies=proxies)

    81     elif not _urlopener:

    82         opener = FancyURLopener()

    83         _urlopener = opener

    84     else:

    85         opener = _urlopener

    86     if data is None:

    87         return opener.open(url)

    88     else:

    89         return opener.open(url, data)

    90 def urlretrieve(url, filename=None, reporthook=None, data=None):

    91     global _urlopener

    92     if not _urlopener:

    93         _urlopener = FancyURLopener()

    94     return _urlopener.retrieve(url, filename, reporthook, data)

    95 def urlcleanup():

    96     if _urlopener:

    97         _urlopener.cleanup()

    98 

    99 # check for SSL

   100 try:

   101     import ssl

   102 except:

   103     _have_ssl = False

   104 else:

   105     _have_ssl = True

   106 

   107 # exception raised when downloaded size does not match content-length

   108 class ContentTooShortError(IOError):

   109     def __init__(self, message, content):

   110         IOError.__init__(self, message)

   111         self.content = content

   112 

   113 ftpcache = {}

   114 class URLopener:

   115     """Class to open URLs.

   116     This is a class rather than just a subroutine because we may need

   117     more than one set of global protocol-specific options.

   118     Note -- this is a base class for those who don't want the

   119     automatic handling of errors type 302 (relocated) and 401

   120     (authorization needed)."""

   121 

   122     __tempfiles = None

   123 

   124     version = "Python-urllib/%s" % __version__

   125 

   126     # Constructor

   127     def __init__(self, proxies=None, **x509):

   128         if proxies is None:

   129             proxies = getproxies()

   130         assert hasattr(proxies, 'has_key'), "proxies must be a mapping"

   131         self.proxies = proxies

   132         self.key_file = x509.get('key_file')

   133         self.cert_file = x509.get('cert_file')

   134         self.addheaders = [('User-Agent', self.version)]

   135         self.__tempfiles = []

   136         self.__unlink = os.unlink # See cleanup()

   137         self.tempcache = None

   138         # Undocumented feature: if you assign {} to tempcache,

   139         # it is used to cache files retrieved with

   140         # self.retrieve().  This is not enabled by default

   141         # since it does not work for changing documents (and I

   142         # haven't got the logic to check expiration headers

   143         # yet).

   144         self.ftpcache = ftpcache

   145         # Undocumented feature: you can use a different

   146         # ftp cache by assigning to the .ftpcache member;

   147         # in case you want logically independent URL openers

   148         # XXX This is not threadsafe.  Bah.

   149 

   150     def __del__(self):

   151         self.close()

   152 

   153     def close(self):

   154         self.cleanup()

   155 

   156     def cleanup(self):

   157         # This code sometimes runs when the rest of this module

   158         # has already been deleted, so it can't use any globals

   159         # or import anything.

   160         if self.__tempfiles:

   161             for file in self.__tempfiles:

   162                 try:

   163                     self.__unlink(file)

   164                 except OSError:

   165                     pass

   166             del self.__tempfiles[:]

   167         if self.tempcache:

   168             self.tempcache.clear()

   169 

   170     def addheader(self, *args):

   171         """Add a header to be used by the HTTP interface only

   172         e.g. u.addheader('Accept', 'sound/basic')"""

   173         self.addheaders.append(args)

   174 

   175     # External interface

   176     def open(self, fullurl, data=None):

   177         """Use URLopener().open(file) instead of open(file, 'r')."""

   178         fullurl = unwrap(toBytes(fullurl))

   179         if self.tempcache and fullurl in self.tempcache:

   180             filename, headers = self.tempcache[fullurl]

   181             fp = open(filename, 'rb')

   182             return addinfourl(fp, headers, fullurl)

   183         urltype, url = splittype(fullurl)

   184         if not urltype:

   185             urltype = 'file'

   186         if urltype in self.proxies:

   187             proxy = self.proxies[urltype]

   188             urltype, proxyhost = splittype(proxy)

   189             host, selector = splithost(proxyhost)

   190             url = (host, fullurl) # Signal special case to open_*()

   191         else:

   192             proxy = None

   193         name = 'open_' + urltype

   194         self.type = urltype

   195         name = name.replace('-', '_')

   196         if not hasattr(self, name):

   197             if proxy:

   198                 return self.open_unknown_proxy(proxy, fullurl, data)

   199             else:

   200                 return self.open_unknown(fullurl, data)

   201         try:

   202             if data is None:

   203                 return getattr(self, name)(url)

   204             else:

   205                 return getattr(self, name)(url, data)

   206         except socket.error, msg:

   207             raise IOError, ('socket error', msg), sys.exc_info()[2]

   208 

   209     def open_unknown(self, fullurl, data=None):

   210         """Overridable interface to open unknown URL type."""

   211         type, url = splittype(fullurl)

   212         raise IOError, ('url error', 'unknown url type', type)

   213 

   214     def open_unknown_proxy(self, proxy, fullurl, data=None):

   215         """Overridable interface to open unknown URL type."""

   216         type, url = splittype(fullurl)

   217         raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)

   218 

   219     # External interface

   220     def retrieve(self, url, filename=None, reporthook=None, data=None):

   221         """retrieve(url) returns (filename, headers) for a local object

   222         or (tempfilename, headers) for a remote object."""

   223         url = unwrap(toBytes(url))

   224         if self.tempcache and url in self.tempcache:

   225             return self.tempcache[url]

   226         type, url1 = splittype(url)

   227         if filename is None and (not type or type == 'file'):

   228             try:

   229                 fp = self.open_local_file(url1)

   230                 hdrs = fp.info()

   231                 del fp

   232                 return url2pathname(splithost(url1)[1]), hdrs

   233             except IOError, msg:

   234                 pass

   235         fp = self.open(url, data)

   236         headers = fp.info()

   237         if filename:

   238             tfp = open(filename, 'wb')

   239         else:

   240             import tempfile

   241             garbage, path = splittype(url)

   242             garbage, path = splithost(path or "")

   243             path, garbage = splitquery(path or "")

   244             path, garbage = splitattr(path or "")

   245             suffix = os.path.splitext(path)[1]

   246             (fd, filename) = tempfile.mkstemp(suffix)

   247             self.__tempfiles.append(filename)

   248             tfp = os.fdopen(fd, 'wb')

   249         result = filename, headers

   250         if self.tempcache is not None:

   251             self.tempcache[url] = result

   252         bs = 1024*8

   253         size = -1

   254         read = 0

   255         blocknum = 0

   256         if reporthook:

   257             if "content-length" in headers:

   258                 size = int(headers["Content-Length"])

   259             reporthook(blocknum, bs, size)

   260         while 1:

   261             block = fp.read(bs)

   262             if block == "":

   263                 break

   264             read += len(block)

   265             tfp.write(block)

   266             blocknum += 1

   267             if reporthook:

   268                 reporthook(blocknum, bs, size)

   269         fp.close()

   270         tfp.close()

   271         del fp

   272         del tfp

   273 

   274         # raise exception if actual size does not match content-length header

   275         if size >= 0 and read < size:

   276             raise ContentTooShortError("retrieval incomplete: got only %i out "

   277                                        "of %i bytes" % (read, size), result)

   278 

   279         return result

   280 

   281     # Each method named open_<type> knows how to open that type of URL

   282 

   283     def open_http(self, url, data=None):

   284         """Use HTTP protocol."""

   285         import httplib

   286         user_passwd = None

   287         proxy_passwd= None

   288         if isinstance(url, str):

   289             host, selector = splithost(url)

   290             if host:

   291                 user_passwd, host = splituser(host)

   292                 host = unquote(host)

   293             realhost = host

   294         else:

   295             host, selector = url

   296             # check whether the proxy contains authorization information

   297             proxy_passwd, host = splituser(host)

   298             # now we proceed with the url we want to obtain

   299             urltype, rest = splittype(selector)

   300             url = rest

   301             user_passwd = None

   302             if urltype.lower() != 'http':

   303                 realhost = None

   304             else:

   305                 realhost, rest = splithost(rest)

   306                 if realhost:

   307                     user_passwd, realhost = splituser(realhost)

   308                 if user_passwd:

   309                     selector = "%s://%s%s" % (urltype, realhost, rest)

   310                 if proxy_bypass(realhost):

   311                     host = realhost

   312 

   313             #print "proxy via http:", host, selector

   314         if not host: raise IOError, ('http error', 'no host given')

   315 

   316         if proxy_passwd:

   317             import base64

   318             proxy_auth = base64.b64encode(proxy_passwd).strip()

   319         else:

   320             proxy_auth = None

   321 

   322         if user_passwd:

   323             import base64

   324             auth = base64.b64encode(user_passwd).strip()

   325         else:

   326             auth = None

   327         h = httplib.HTTP(host)

   328         if data is not None:

   329             h.putrequest('POST', selector)

   330             h.putheader('Content-Type', 'application/x-www-form-urlencoded')

   331             h.putheader('Content-Length', '%d' % len(data))

   332         else:

   333             h.putrequest('GET', selector)

   334         if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)

   335         if auth: h.putheader('Authorization', 'Basic %s' % auth)

   336         if realhost: h.putheader('Host', realhost)

   337         for args in self.addheaders: h.putheader(*args)

   338         h.endheaders()

   339         if data is not None:

   340             h.send(data)

   341         errcode, errmsg, headers = h.getreply()

   342         fp = h.getfile()

   343         if errcode == -1:

   344             if fp: fp.close()

   345             # something went wrong with the HTTP status line

   346             raise IOError, ('http protocol error', 0,

   347                             'got a bad status line', None)

   348         # According to RFC 2616, "2xx" code indicates that the client's

   349         # request was successfully received, understood, and accepted.

   350         if (200 <= errcode < 300):

   351             return addinfourl(fp, headers, "http:" + url, errcode)

   352         else:

   353             if data is None:

   354                 return self.http_error(url, fp, errcode, errmsg, headers)

   355             else:

   356                 return self.http_error(url, fp, errcode, errmsg, headers, data)

   357 

   358     def http_error(self, url, fp, errcode, errmsg, headers, data=None):

   359         """Handle http errors.

   360         Derived class can override this, or provide specific handlers

   361         named http_error_DDD where DDD is the 3-digit error code."""

   362         # First check if there's a specific handler for this error

   363         name = 'http_error_%d' % errcode

   364         if hasattr(self, name):

   365             method = getattr(self, name)

   366             if data is None:

   367                 result = method(url, fp, errcode, errmsg, headers)

   368             else:

   369                 result = method(url, fp, errcode, errmsg, headers, data)

   370             if result: return result

   371         return self.http_error_default(url, fp, errcode, errmsg, headers)

   372 

   373     def http_error_default(self, url, fp, errcode, errmsg, headers):

   374         """Default error handler: close the connection and raise IOError."""

   375         void = fp.read()

   376         fp.close()

   377         raise IOError, ('http error', errcode, errmsg, headers)

   378 

   379     if _have_ssl:

   380         def open_https(self, url, data=None):

   381             """Use HTTPS protocol."""

   382 

   383             import httplib

   384             user_passwd = None

   385             proxy_passwd = None

   386             if isinstance(url, str):

   387                 host, selector = splithost(url)

   388                 if host:

   389                     user_passwd, host = splituser(host)

   390                     host = unquote(host)

   391                 realhost = host

   392             else:

   393                 host, selector = url

   394                 # here, we determine, whether the proxy contains authorization information

   395                 proxy_passwd, host = splituser(host)

   396                 urltype, rest = splittype(selector)

   397                 url = rest

   398                 user_passwd = None

   399                 if urltype.lower() != 'https':

   400                     realhost = None

   401                 else:

   402                     realhost, rest = splithost(rest)

   403                     if realhost:

   404                         user_passwd, realhost = splituser(realhost)

   405                     if user_passwd:

   406                         selector = "%s://%s%s" % (urltype, realhost, rest)

   407                 #print "proxy via https:", host, selector

   408             if not host: raise IOError, ('https error', 'no host given')

   409             if proxy_passwd:

   410                 import base64

   411                 proxy_auth = base64.b64encode(proxy_passwd).strip()

   412             else:

   413                 proxy_auth = None

   414             if user_passwd:

   415                 import base64

   416                 auth = base64.b64encode(user_passwd).strip()

   417             else:

   418                 auth = None

   419             h = httplib.HTTPS(host, 0,

   420                               key_file=self.key_file,

   421                               cert_file=self.cert_file)

   422             if data is not None:

   423                 h.putrequest('POST', selector)

   424                 h.putheader('Content-Type',

   425                             'application/x-www-form-urlencoded')

   426                 h.putheader('Content-Length', '%d' % len(data))

   427             else:

   428                 h.putrequest('GET', selector)

   429             if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)

   430             if auth: h.putheader('Authorization', 'Basic %s' % auth)

   431             if realhost: h.putheader('Host', realhost)

   432             for args in self.addheaders: h.putheader(*args)

   433             h.endheaders()

   434             if data is not None:

   435                 h.send(data)

   436             errcode, errmsg, headers = h.getreply()

   437             fp = h.getfile()

   438             if errcode == -1:

   439                 if fp: fp.close()

   440                 # something went wrong with the HTTP status line

   441                 raise IOError, ('http protocol error', 0,

   442                                 'got a bad status line', None)

   443             # According to RFC 2616, "2xx" code indicates that the client's

   444             # request was successfully received, understood, and accepted.

   445             if (200 <= errcode < 300):

   446                 return addinfourl(fp, headers, "https:" + url, errcode)

   447             else:

   448                 if data is None:

   449                     return self.http_error(url, fp, errcode, errmsg, headers)

   450                 else:

   451                     return self.http_error(url, fp, errcode, errmsg, headers,

   452                                            data)

   453 

   454     def open_file(self, url):

   455         """Use local file or FTP depending on form of URL."""

   456         if not isinstance(url, str):

   457             raise IOError, ('file error', 'proxy support for file protocol currently not implemented')

   458         if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':

   459             return self.open_ftp(url)

   460         else:

   461             return self.open_local_file(url)

   462 

   463     def open_local_file(self, url):

   464         """Use local file."""

   465         import mimetypes, mimetools, email.utils

   466         try:

   467             from cStringIO import StringIO

   468         except ImportError:

   469             from StringIO import StringIO

   470         host, file = splithost(url)

   471         localname = url2pathname(file)

   472         try:

   473             stats = os.stat(localname)

   474         except OSError, e:

   475             raise IOError(e.errno, e.strerror, e.filename)

   476         size = stats.st_size

   477         modified = email.utils.formatdate(stats.st_mtime, usegmt=True)

   478         mtype = mimetypes.guess_type(url)[0]

   479         headers = mimetools.Message(StringIO(

   480             'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %

   481             (mtype or 'text/plain', size, modified)))

   482         if not host:

   483             urlfile = file

   484             if file[:1] == '/':

   485                 urlfile = 'file://' + file

   486             return addinfourl(open(localname, 'rb'),

   487                               headers, urlfile)

   488         host, port = splitport(host)

   489         if not port \

   490            and socket.gethostbyname(host) in (localhost(), thishost()):

   491             urlfile = file

   492             if file[:1] == '/':

   493                 urlfile = 'file://' + file

   494             return addinfourl(open(localname, 'rb'),

   495                               headers, urlfile)

   496         raise IOError, ('local file error', 'not on local host')

   497 

   498     def open_ftp(self, url):

   499         """Use FTP protocol."""

   500         if not isinstance(url, str):

   501             raise IOError, ('ftp error', 'proxy support for ftp protocol currently not implemented')

   502         import mimetypes, mimetools

   503         try:

   504             from cStringIO import StringIO

   505         except ImportError:

   506             from StringIO import StringIO

   507         host, path = splithost(url)

   508         if not host: raise IOError, ('ftp error', 'no host given')

   509         host, port = splitport(host)

   510         user, host = splituser(host)

   511         if user: user, passwd = splitpasswd(user)

   512         else: passwd = None

   513         host = unquote(host)

   514         user = unquote(user or '')

   515         passwd = unquote(passwd or '')

   516         host = socket.gethostbyname(host)

   517         if not port:

   518             import ftplib

   519             port = ftplib.FTP_PORT

   520         else:

   521             port = int(port)

   522         path, attrs = splitattr(path)

   523         path = unquote(path)

   524         dirs = path.split('/')

   525         dirs, file = dirs[:-1], dirs[-1]

   526         if dirs and not dirs[0]: dirs = dirs[1:]

   527         if dirs and not dirs[0]: dirs[0] = '/'

   528         key = user, host, port, '/'.join(dirs)

   529         # XXX thread unsafe!

   530         if len(self.ftpcache) > MAXFTPCACHE:

   531             # Prune the cache, rather arbitrarily

   532             for k in self.ftpcache.keys():

   533                 if k != key:

   534                     v = self.ftpcache[k]

   535                     del self.ftpcache[k]

   536                     v.close()

   537         try:

   538             if not key in self.ftpcache:

   539                 self.ftpcache[key] = \

   540                     ftpwrapper(user, passwd, host, port, dirs)

   541             if not file: type = 'D'

   542             else: type = 'I'

   543             for attr in attrs:

   544                 attr, value = splitvalue(attr)

   545                 if attr.lower() == 'type' and \

   546                    value in ('a', 'A', 'i', 'I', 'd', 'D'):

   547                     type = value.upper()

   548             (fp, retrlen) = self.ftpcache[key].retrfile(file, type)

   549             mtype = mimetypes.guess_type("ftp:" + url)[0]

   550             headers = ""

   551             if mtype:

   552                 headers += "Content-Type: %s\n" % mtype

   553             if retrlen is not None and retrlen >= 0:

   554                 headers += "Content-Length: %d\n" % retrlen

   555             headers = mimetools.Message(StringIO(headers))

   556             return addinfourl(fp, headers, "ftp:" + url)

   557         except ftperrors(), msg:

   558             raise IOError, ('ftp error', msg), sys.exc_info()[2]

   559 

   560     def open_data(self, url, data=None):

   561         """Use "data" URL."""

   562         if not isinstance(url, str):

   563             raise IOError, ('data error', 'proxy support for data protocol currently not implemented')

   564         # ignore POSTed data

   565         #

   566         # syntax of data URLs:

   567         # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data

   568         # mediatype := [ type "/" subtype ] *( ";" parameter )

   569         # data      := *urlchar

   570         # parameter := attribute "=" value

   571         import mimetools

   572         try:

   573             from cStringIO import StringIO

   574         except ImportError:

   575             from StringIO import StringIO

   576         try:

   577             [type, data] = url.split(',', 1)

   578         except ValueError:

   579             raise IOError, ('data error', 'bad data URL')

   580         if not type:

   581             type = 'text/plain;charset=US-ASCII'

   582         semi = type.rfind(';')

   583         if semi >= 0 and '=' not in type[semi:]:

   584             encoding = type[semi+1:]

   585             type = type[:semi]

   586         else:

   587             encoding = ''

   588         msg = []

   589         msg.append('Date: %s'%time.strftime('%a, %d %b %Y %T GMT',

   590                                             time.gmtime(time.time())))

   591         msg.append('Content-type: %s' % type)

   592         if encoding == 'base64':

   593             import base64

   594             data = base64.decodestring(data)

   595         else:

   596             data = unquote(data)

   597         msg.append('Content-Length: %d' % len(data))

   598         msg.append('')

   599         msg.append(data)

   600         msg = '\n'.join(msg)

   601         f = StringIO(msg)

   602         headers = mimetools.Message(f, 0)

   603         #f.fileno = None     # needed for addinfourl

   604         return addinfourl(f, headers, url)

   605 

   606 

   607 class FancyURLopener(URLopener):

   608     """Derived class with handlers for errors we can handle (perhaps)."""

   609 

   610     def __init__(self, *args, **kwargs):

   611         URLopener.__init__(self, *args, **kwargs)

   612         self.auth_cache = {}

   613         self.tries = 0

   614         self.maxtries = 10

   615 

   616     def http_error_default(self, url, fp, errcode, errmsg, headers):

   617         """Default error handling -- don't raise an exception."""

   618         return addinfourl(fp, headers, "http:" + url, errcode)

   619 

   620     def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):

   621         """Error 302 -- relocated (temporarily)."""

   622         self.tries += 1

   623         if self.maxtries and self.tries >= self.maxtries:

   624             if hasattr(self, "http_error_500"):

   625                 meth = self.http_error_500

   626             else:

   627                 meth = self.http_error_default

   628             self.tries = 0

   629             return meth(url, fp, 500,

   630                         "Internal Server Error: Redirect Recursion", headers)

   631         result = self.redirect_internal(url, fp, errcode, errmsg, headers,

   632                                         data)

   633         self.tries = 0

   634         return result

   635 

   636     def redirect_internal(self, url, fp, errcode, errmsg, headers, data):

   637         if 'location' in headers:

   638             newurl = headers['location']

   639         elif 'uri' in headers:

   640             newurl = headers['uri']

   641         else:

   642             return

   643         void = fp.read()

   644         fp.close()

   645         # In case the server sent a relative URL, join with original:

   646         newurl = basejoin(self.type + ":" + url, newurl)

   647         return self.open(newurl)

   648 

   649     def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):

   650         """Error 301 -- also relocated (permanently)."""

   651         return self.http_error_302(url, fp, errcode, errmsg, headers, data)

   652 

   653     def http_error_303(self, url, fp, errcode, errmsg, headers, data=None):

   654         """Error 303 -- also relocated (essentially identical to 302)."""

   655         return self.http_error_302(url, fp, errcode, errmsg, headers, data)

   656 

   657     def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):

   658         """Error 307 -- relocated, but turn POST into error."""

   659         if data is None:

   660             return self.http_error_302(url, fp, errcode, errmsg, headers, data)

   661         else:

   662             return self.http_error_default(url, fp, errcode, errmsg, headers)

   663 

   664     def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):

   665         """Error 401 -- authentication required.

   666         This function supports Basic authentication only."""

   667         if not 'www-authenticate' in headers:

   668             URLopener.http_error_default(self, url, fp,

   669                                          errcode, errmsg, headers)

   670         stuff = headers['www-authenticate']

   671         import re

   672         match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)

   673         if not match:

   674             URLopener.http_error_default(self, url, fp,

   675                                          errcode, errmsg, headers)

   676         scheme, realm = match.groups()

   677         if scheme.lower() != 'basic':

   678             URLopener.http_error_default(self, url, fp,

   679                                          errcode, errmsg, headers)

   680         name = 'retry_' + self.type + '_basic_auth'

   681         if data is None:

   682             return getattr(self,name)(url, realm)

   683         else:

   684             return getattr(self,name)(url, realm, data)

   685 

   686     def http_error_407(self, url, fp, errcode, errmsg, headers, data=None):

   687         """Error 407 -- proxy authentication required.

   688         This function supports Basic authentication only."""

   689         if not 'proxy-authenticate' in headers:

   690             URLopener.http_error_default(self, url, fp,

   691                                          errcode, errmsg, headers)

   692         stuff = headers['proxy-authenticate']

   693         import re

   694         match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)

   695         if not match:

   696             URLopener.http_error_default(self, url, fp,

   697                                          errcode, errmsg, headers)

   698         scheme, realm = match.groups()

   699         if scheme.lower() != 'basic':

   700             URLopener.http_error_default(self, url, fp,

   701                                          errcode, errmsg, headers)

   702         name = 'retry_proxy_' + self.type + '_basic_auth'

   703         if data is None:

   704             return getattr(self,name)(url, realm)

   705         else:

   706             return getattr(self,name)(url, realm, data)

   707 

   708     def retry_proxy_http_basic_auth(self, url, realm, data=None):

   709         host, selector = splithost(url)

   710         newurl = 'http://' + host + selector

   711         proxy = self.proxies['http']

   712         urltype, proxyhost = splittype(proxy)

   713         proxyhost, proxyselector = splithost(proxyhost)

   714         i = proxyhost.find('@') + 1

   715         proxyhost = proxyhost[i:]

   716         user, passwd = self.get_user_passwd(proxyhost, realm, i)

   717         if not (user or passwd): return None

   718         proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost

   719         self.proxies['http'] = 'http://' + proxyhost + proxyselector

   720         if data is None:

   721             return self.open(newurl)

   722         else:

   723             return self.open(newurl, data)

   724 

   725     def retry_proxy_https_basic_auth(self, url, realm, data=None):

   726         host, selector = splithost(url)

   727         newurl = 'https://' + host + selector

   728         proxy = self.proxies['https']

   729         urltype, proxyhost = splittype(proxy)

   730         proxyhost, proxyselector = splithost(proxyhost)

   731         i = proxyhost.find('@') + 1

   732         proxyhost = proxyhost[i:]

   733         user, passwd = self.get_user_passwd(proxyhost, realm, i)

   734         if not (user or passwd): return None

   735         proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost

   736         self.proxies['https'] = 'https://' + proxyhost + proxyselector

   737         if data is None:

   738             return self.open(newurl)

   739         else:

   740             return self.open(newurl, data)

   741 

   742     def retry_http_basic_auth(self, url, realm, data=None):

   743         host, selector = splithost(url)

   744         i = host.find('@') + 1

   745         host = host[i:]

   746         user, passwd = self.get_user_passwd(host, realm, i)

   747         if not (user or passwd): return None

   748         host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host

   749         newurl = 'http://' + host + selector

   750         if data is None:

   751             return self.open(newurl)

   752         else:

   753             return self.open(newurl, data)

   754 

   755     def retry_https_basic_auth(self, url, realm, data=None):

   756         host, selector = splithost(url)

   757         i = host.find('@') + 1

   758         host = host[i:]

   759         user, passwd = self.get_user_passwd(host, realm, i)

   760         if not (user or passwd): return None

   761         host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host

   762         newurl = 'https://' + host + selector

   763         if data is None:

   764             return self.open(newurl)

   765         else:

   766             return self.open(newurl, data)

   767 

   768     def get_user_passwd(self, host, realm, clear_cache = 0):

   769         key = realm + '@' + host.lower()

   770         if key in self.auth_cache:

   771             if clear_cache:

   772                 del self.auth_cache[key]

   773             else:

   774                 return self.auth_cache[key]

   775         user, passwd = self.prompt_user_passwd(host, realm)

   776         if user or passwd: self.auth_cache[key] = (user, passwd)

   777         return user, passwd

   778 

   779     def prompt_user_passwd(self, host, realm):

   780         """Override this in a GUI environment!"""

   781         import getpass

   782         try:

   783             user = raw_input("Enter username for %s at %s: " % (realm,

   784                                                                 host))

   785             passwd = getpass.getpass("Enter password for %s in %s at %s: " %

   786                 (user, realm, host))

   787             return user, passwd

   788         except KeyboardInterrupt:

   789             print

   790             return None, None

   791 

   792 

   793 # Utility functions

   794 

   795 _localhost = None

   796 def localhost():

   797     """Return the IP address of the magic hostname 'localhost'."""

   798     global _localhost

   799     if _localhost is None:

   800         _localhost = socket.gethostbyname('localhost')

   801     return _localhost

   802 

   803 _thishost = None

   804 def thishost():

   805     """Return the IP address of the current host."""

   806     global _thishost

   807     if _thishost is None:

   808         _thishost = socket.gethostbyname(socket.gethostname())

   809     return _thishost

   810 

   811 _ftperrors = None

   812 def ftperrors():

   813     """Return the set of errors raised by the FTP class."""

   814     global _ftperrors

   815     if _ftperrors is None:

   816         import ftplib

   817         _ftperrors = ftplib.all_errors

   818     return _ftperrors

   819 

   820 _noheaders = None

   821 def noheaders():

   822     """Return an empty mimetools.Message object."""

   823     global _noheaders

   824     if _noheaders is None:

   825         import mimetools

   826         try:

   827             from cStringIO import StringIO

   828         except ImportError:

   829             from StringIO import StringIO

   830         _noheaders = mimetools.Message(StringIO(), 0)

   831         _noheaders.fp.close()   # Recycle file descriptor

   832     return _noheaders

   833 

   834 

   835 # Utility classes

   836 

   837 class ftpwrapper:

   838     """Class used by open_ftp() for cache of open FTP connections."""

   839 

   840     def __init__(self, user, passwd, host, port, dirs,

   841                  timeout=socket._GLOBAL_DEFAULT_TIMEOUT):

   842         self.user = user

   843         self.passwd = passwd

   844         self.host = host

   845         self.port = port

   846         self.dirs = dirs

   847         self.timeout = timeout

   848         self.init()

   849 

   850     def init(self):

   851         import ftplib

   852         self.busy = 0

   853         self.ftp = ftplib.FTP()

   854         self.ftp.connect(self.host, self.port, self.timeout)

   855         self.ftp.login(self.user, self.passwd)

   856         for dir in self.dirs:

   857             self.ftp.cwd(dir)

   858 

   859     def retrfile(self, file, type):

   860         import ftplib

   861         self.endtransfer()

   862         if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1

   863         else: cmd = 'TYPE ' + type; isdir = 0

   864         try:

   865             self.ftp.voidcmd(cmd)

   866         except ftplib.all_errors:

   867             self.init()

   868             self.ftp.voidcmd(cmd)

   869         conn = None

   870         if file and not isdir:

   871             # Try to retrieve as a file

   872             try:

   873                 cmd = 'RETR ' + file

   874                 conn = self.ftp.ntransfercmd(cmd)

   875             except ftplib.error_perm, reason:

   876                 if str(reason)[:3] != '550':

   877                     raise IOError, ('ftp error', reason), sys.exc_info()[2]

   878         if not conn:

   879             # Set transfer mode to ASCII!

   880             self.ftp.voidcmd('TYPE A')

   881             # Try a directory listing. Verify that directory exists.

   882             if file:

   883                 pwd = self.ftp.pwd()

   884                 try:

   885                     try:

   886                         self.ftp.cwd(file)

   887                     except ftplib.error_perm, reason:

   888                         raise IOError, ('ftp error', reason), sys.exc_info()[2]

   889                 finally:

   890                     self.ftp.cwd(pwd)

   891                 cmd = 'LIST ' + file

   892             else:

   893                 cmd = 'LIST'

   894             conn = self.ftp.ntransfercmd(cmd)

   895         self.busy = 1

   896         # Pass back both a suitably decorated object and a retrieval length

   897         return (addclosehook(conn[0].makefile('rb'),

   898                              self.endtransfer), conn[1])

   899     def endtransfer(self):

   900         if not self.busy:

   901             return

   902         self.busy = 0

   903         try:

   904             self.ftp.voidresp()

   905         except ftperrors():

   906             pass

   907 

   908     def close(self):

   909         self.endtransfer()

   910         try:

   911             self.ftp.close()

   912         except ftperrors():

   913             pass

   914 

   915 class addbase:

   916     """Base class for addinfo and addclosehook."""

   917 

   918     def __init__(self, fp):

   919         self.fp = fp

   920         self.read = self.fp.read

   921         self.readline = self.fp.readline

   922         if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines

   923         if hasattr(self.fp, "fileno"):

   924             self.fileno = self.fp.fileno

   925         else:

   926             self.fileno = lambda: None

   927         if hasattr(self.fp, "__iter__"):

   928             self.__iter__ = self.fp.__iter__

   929             if hasattr(self.fp, "next"):

   930                 self.next = self.fp.next

   931 

   932     def __repr__(self):

   933         return '<%s at %r whose fp = %r>' % (self.__class__.__name__,

   934                                              id(self), self.fp)

   935 

   936     def close(self):

   937         self.read = None

   938         self.readline = None

   939         self.readlines = None

   940         self.fileno = None

   941         if self.fp: self.fp.close()

   942         self.fp = None

   943 

   944 class addclosehook(addbase):

   945     """Class to add a close hook to an open file."""

   946 

   947     def __init__(self, fp, closehook, *hookargs):

   948         addbase.__init__(self, fp)

   949         self.closehook = closehook

   950         self.hookargs = hookargs

   951 

   952     def close(self):

   953         addbase.close(self)

   954         if self.closehook:

   955             self.closehook(*self.hookargs)

   956             self.closehook = None

   957             self.hookargs = None

   958 

   959 class addinfo(addbase):

   960     """class to add an info() method to an open file."""

   961 

   962     def __init__(self, fp, headers):

   963         addbase.__init__(self, fp)

   964         self.headers = headers

   965 

   966     def info(self):

   967         return self.headers

   968 

   969 class addinfourl(addbase):

   970     """class to add info() and geturl() methods to an open file."""

   971 

   972     def __init__(self, fp, headers, url, code=None):

   973         addbase.__init__(self, fp)

   974         self.headers = headers

   975         self.url = url

   976         self.code = code

   977 

   978     def info(self):

   979         return self.headers

   980 

   981     def getcode(self):

   982         return self.code

   983 

   984     def geturl(self):

   985         return self.url

   986 

   987 

   988 # Utilities to parse URLs (most of these return None for missing parts):

   989 # unwrap('<URL:type://host/path>') --> 'type://host/path'

   990 # splittype('type:opaquestring') --> 'type', 'opaquestring'

   991 # splithost('//host[:port]/path') --> 'host[:port]', '/path'

   992 # splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'

   993 # splitpasswd('user:passwd') -> 'user', 'passwd'

   994 # splitport('host:port') --> 'host', 'port'

   995 # splitquery('/path?query') --> '/path', 'query'

   996 # splittag('/path#tag') --> '/path', 'tag'

   997 # splitattr('/path;attr1=value1;attr2=value2;...') ->

   998 #   '/path', ['attr1=value1', 'attr2=value2', ...]

   999 # splitvalue('attr=value') --> 'attr', 'value'

  1000 # unquote('abc%20def') -> 'abc def'

  1001 # quote('abc def') -> 'abc%20def')

  1002 

  1003 try:

  1004     unicode

  1005 except NameError:

  1006     def _is_unicode(x):

  1007         return 0

  1008 else:

  1009     def _is_unicode(x):

  1010         return isinstance(x, unicode)

  1011 

  1012 def toBytes(url):

  1013     """toBytes(u"URL") --> 'URL'."""

  1014     # Most URL schemes require ASCII. If that changes, the conversion

  1015     # can be relaxed

  1016     if _is_unicode(url):

  1017         try:

  1018             url = url.encode("ASCII")

  1019         except UnicodeError:

  1020             raise UnicodeError("URL " + repr(url) +

  1021                                " contains non-ASCII characters")

  1022     return url

  1023 

  1024 def unwrap(url):

  1025     """unwrap('<URL:type://host/path>') --> 'type://host/path'."""

  1026     url = url.strip()

  1027     if url[:1] == '<' and url[-1:] == '>':

  1028         url = url[1:-1].strip()

  1029     if url[:4] == 'URL:': url = url[4:].strip()

  1030     return url

  1031 

  1032 _typeprog = None

  1033 def splittype(url):

  1034     """splittype('type:opaquestring') --> 'type', 'opaquestring'."""

  1035     global _typeprog

  1036     if _typeprog is None:

  1037         import re

  1038         _typeprog = re.compile('^([^/:]+):')

  1039 

  1040     match = _typeprog.match(url)

  1041     if match:

  1042         scheme = match.group(1)

  1043         return scheme.lower(), url[len(scheme) + 1:]

  1044     return None, url

  1045 

  1046 _hostprog = None

  1047 def splithost(url):

  1048     """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""

  1049     global _hostprog

  1050     if _hostprog is None:

  1051         import re

  1052         _hostprog = re.compile('^//([^/?]*)(.*)$')

  1053 

  1054     match = _hostprog.match(url)

  1055     if match: return match.group(1, 2)

  1056     return None, url

  1057 

  1058 _userprog = None

  1059 def splituser(host):

  1060     """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""

  1061     global _userprog

  1062     if _userprog is None:

  1063         import re

  1064         _userprog = re.compile('^(.*)@(.*)$')

  1065 

  1066     match = _userprog.match(host)

  1067     if match: return map(unquote, match.group(1, 2))

  1068     return None, host

  1069 

  1070 _passwdprog = None

  1071 def splitpasswd(user):

  1072     """splitpasswd('user:passwd') -> 'user', 'passwd'."""

  1073     global _passwdprog

  1074     if _passwdprog is None:

  1075         import re

  1076         _passwdprog = re.compile('^([^:]*):(.*)$')

  1077 

  1078     match = _passwdprog.match(user)

  1079     if match: return match.group(1, 2)

  1080     return user, None

  1081 

  1082 # splittag('/path#tag') --> '/path', 'tag'

  1083 _portprog = None

  1084 def splitport(host):

  1085     """splitport('host:port') --> 'host', 'port'."""

  1086     global _portprog

  1087     if _portprog is None:

  1088         import re

  1089         _portprog = re.compile('^(.*):([0-9]+)$')

  1090 

  1091     match = _portprog.match(host)

  1092     if match: return match.group(1, 2)

  1093     return host, None

  1094 

  1095 _nportprog = None

  1096 def splitnport(host, defport=-1):

  1097     """Split host and port, returning numeric port.

  1098     Return given default port if no ':' found; defaults to -1.

  1099     Return numerical port if a valid number are found after ':'.

  1100     Return None if ':' but not a valid number."""

  1101     global _nportprog

  1102     if _nportprog is None:

  1103         import re

  1104         _nportprog = re.compile('^(.*):(.*)$')

  1105 

  1106     match = _nportprog.match(host)

  1107     if match:

  1108         host, port = match.group(1, 2)

  1109         try:

  1110             if not port: raise ValueError, "no digits"

  1111             nport = int(port)

  1112         except ValueError:

  1113             nport = None

  1114         return host, nport

  1115     return host, defport

  1116 

  1117 _queryprog = None

  1118 def splitquery(url):

  1119     """splitquery('/path?query') --> '/path', 'query'."""

  1120     global _queryprog

  1121     if _queryprog is None:

  1122         import re

  1123         _queryprog = re.compile('^(.*)\?([^?]*)$')

  1124 

  1125     match = _queryprog.match(url)

  1126     if match: return match.group(1, 2)

  1127     return url, None

  1128 

  1129 _tagprog = None

  1130 def splittag(url):

  1131     """splittag('/path#tag') --> '/path', 'tag'."""

  1132     global _tagprog

  1133     if _tagprog is None:

  1134         import re

  1135         _tagprog = re.compile('^(.*)#([^#]*)$')